From 0b75e1ac7643787e201fd62628823e6d51ca6353 Mon Sep 17 00:00:00 2001 From: Gavin McDonald Date: Thu, 28 Oct 2010 02:12:01 +0000 Subject: [PATCH 1/1] Thrift now a TLP - INFRA-3116 git-svn-id: https://svn.apache.org/repos/asf/thrift/branches/0.1.x@1028168 13f79535-47bb-0310-9956-ffa450edef68 --- .gitignore | 88 + CHANGES | 35 + CONTRIBUTORS | 77 + DISCLAIMER | 6 + LICENSE | 202 ++ Makefile.am | 28 + NEWS | 79 + NOTICE | 26 + README | 137 + aclocal/ax_boost_base.m4 | 198 ++ aclocal/ax_javac_and_java.m4 | 84 + aclocal/ax_lib_event.m4 | 173 + aclocal/ax_lib_zlib.m4 | 173 + aclocal/ax_signed_right_shift.m4 | 127 + aclocal/ax_thrift_internal.m4 | 39 + bootstrap.sh | 35 + cleanup.sh | 58 + compiler/cpp/Makefile.am | 136 + compiler/cpp/README | 39 + .../cpp/src/generate/t_cocoa_generator.cc | 2059 +++++++++++ compiler/cpp/src/generate/t_cpp_generator.cc | 3003 ++++++++++++++++ .../cpp/src/generate/t_csharp_generator.cc | 1700 ++++++++++ compiler/cpp/src/generate/t_erl_generator.cc | 932 +++++ compiler/cpp/src/generate/t_generator.cc | 173 + compiler/cpp/src/generate/t_generator.h | 321 ++ compiler/cpp/src/generate/t_hs_generator.cc | 1445 ++++++++ compiler/cpp/src/generate/t_html_generator.cc | 637 ++++ compiler/cpp/src/generate/t_java_generator.cc | 3008 +++++++++++++++++ .../cpp/src/generate/t_ocaml_generator.cc | 1673 +++++++++ compiler/cpp/src/generate/t_oop_generator.h | 77 + compiler/cpp/src/generate/t_perl_generator.cc | 1815 ++++++++++ compiler/cpp/src/generate/t_php_generator.cc | 2281 +++++++++++++ compiler/cpp/src/generate/t_py_generator.cc | 2310 +++++++++++++ compiler/cpp/src/generate/t_rb_generator.cc | 1097 ++++++ compiler/cpp/src/generate/t_st_generator.cc | 1071 ++++++ compiler/cpp/src/generate/t_xsd_generator.cc | 354 ++ compiler/cpp/src/globals.h | 117 + compiler/cpp/src/main.cc | 1207 +++++++ compiler/cpp/src/main.h | 103 + compiler/cpp/src/md5.c | 381 +++ compiler/cpp/src/md5.h | 91 + compiler/cpp/src/parse/t_base_type.h | 137 + compiler/cpp/src/parse/t_const.h | 59 + compiler/cpp/src/parse/t_const_value.h | 121 + compiler/cpp/src/parse/t_container.h | 56 + compiler/cpp/src/parse/t_doc.h | 51 + compiler/cpp/src/parse/t_enum.h | 59 + compiler/cpp/src/parse/t_enum_value.h | 64 + compiler/cpp/src/parse/t_field.h | 150 + compiler/cpp/src/parse/t_function.h | 93 + compiler/cpp/src/parse/t_list.h | 56 + compiler/cpp/src/parse/t_map.h | 64 + compiler/cpp/src/parse/t_program.h | 223 ++ compiler/cpp/src/parse/t_scope.h | 86 + compiler/cpp/src/parse/t_service.h | 68 + compiler/cpp/src/parse/t_set.h | 55 + compiler/cpp/src/parse/t_struct.h | 127 + compiler/cpp/src/parse/t_type.h | 176 + compiler/cpp/src/parse/t_typedef.h | 70 + compiler/cpp/src/platform.h | 36 + compiler/cpp/src/thriftl.ll | 302 ++ compiler/cpp/src/thrifty.yy | 1127 ++++++ configure.ac | 226 ++ contrib/fb303/LICENSE | 16 + contrib/fb303/Makefile.am | 31 + contrib/fb303/README | 37 + contrib/fb303/acinclude.m4 | 258 ++ contrib/fb303/aclocal/ax_boost_base.m4 | 198 ++ contrib/fb303/bootstrap.sh | 26 + contrib/fb303/configure.ac | 115 + contrib/fb303/cpp/FacebookBase.cpp | 124 + contrib/fb303/cpp/FacebookBase.h | 103 + contrib/fb303/cpp/Makefile.am | 84 + contrib/fb303/cpp/ServiceTracker.cpp | 481 +++ contrib/fb303/cpp/ServiceTracker.h | 215 ++ contrib/fb303/global_footer.mk | 21 + contrib/fb303/global_header.mk | 38 + contrib/fb303/if/fb303.thrift | 112 + contrib/fb303/java/FacebookBase.java | 103 + contrib/fb303/java/build.xml | 84 + contrib/fb303/php/FacebookBase.php | 89 + contrib/fb303/py/Makefile.am | 44 + contrib/fb303/py/fb303/FacebookBase.py | 82 + contrib/fb303/py/fb303_scripts/__init__.py | 20 + .../py/fb303_scripts/fb303_simple_mgmt.py | 195 ++ contrib/fb303/py/setup.py | 27 + contrib/thrift.el | 126 + contrib/thrift.spec | 206 ++ contrib/thrift.vim | 91 + contrib/thrift_dump.cpp | 91 + doc/lgpl-2.1.txt | 504 +++ doc/otp-base-license.txt | 20 + doc/thrift.bnf | 96 + doc/thrift.tex | 1057 ++++++ lib/Makefile.am | 55 + lib/cocoa/README | 21 + lib/cocoa/src/TApplicationException.h | 44 + lib/cocoa/src/TApplicationException.m | 130 + lib/cocoa/src/TException.h | 34 + lib/cocoa/src/TException.m | 64 + lib/cocoa/src/TProcessor.h | 28 + lib/cocoa/src/protocol/TBinaryProtocol.h | 51 + lib/cocoa/src/protocol/TBinaryProtocol.m | 469 +++ lib/cocoa/src/protocol/TProtocol.h | 148 + lib/cocoa/src/protocol/TProtocolException.h | 25 + lib/cocoa/src/protocol/TProtocolException.m | 23 + lib/cocoa/src/protocol/TProtocolFactory.h | 29 + lib/cocoa/src/protocol/TProtocolUtil.h | 29 + lib/cocoa/src/protocol/TProtocolUtil.m | 104 + lib/cocoa/src/server/TSocketServer.h | 40 + lib/cocoa/src/server/TSocketServer.m | 114 + lib/cocoa/src/transport/THTTPClient.h | 42 + lib/cocoa/src/transport/THTTPClient.m | 159 + .../src/transport/TNSFileHandleTransport.h | 35 + .../src/transport/TNSFileHandleTransport.m | 87 + lib/cocoa/src/transport/TNSStreamTransport.h | 38 + lib/cocoa/src/transport/TNSStreamTransport.m | 87 + lib/cocoa/src/transport/TSocketClient.h | 32 + lib/cocoa/src/transport/TSocketClient.m | 47 + lib/cocoa/src/transport/TTransport.h | 36 + lib/cocoa/src/transport/TTransportException.h | 30 + lib/cocoa/src/transport/TTransportException.m | 43 + lib/cpp/Makefile.am | 158 + lib/cpp/README | 67 + lib/cpp/src/TLogging.h | 163 + lib/cpp/src/TProcessor.h | 53 + lib/cpp/src/TReflectionLocal.h | 96 + lib/cpp/src/Thrift.cpp | 148 + lib/cpp/src/Thrift.h | 190 ++ lib/cpp/src/concurrency/Exception.h | 60 + lib/cpp/src/concurrency/FunctionRunner.h | 77 + lib/cpp/src/concurrency/Monitor.cpp | 137 + lib/cpp/src/concurrency/Monitor.h | 84 + lib/cpp/src/concurrency/Mutex.cpp | 160 + lib/cpp/src/concurrency/Mutex.h | 112 + .../src/concurrency/PosixThreadFactory.cpp | 308 ++ lib/cpp/src/concurrency/PosixThreadFactory.h | 130 + lib/cpp/src/concurrency/Thread.h | 125 + lib/cpp/src/concurrency/ThreadManager.cpp | 493 +++ lib/cpp/src/concurrency/ThreadManager.h | 169 + lib/cpp/src/concurrency/TimerManager.cpp | 284 ++ lib/cpp/src/concurrency/TimerManager.h | 120 + lib/cpp/src/concurrency/Util.cpp | 55 + lib/cpp/src/concurrency/Util.h | 100 + lib/cpp/src/concurrency/test/Tests.cpp | 155 + .../src/concurrency/test/ThreadFactoryTests.h | 357 ++ .../src/concurrency/test/ThreadManagerTests.h | 366 ++ .../src/concurrency/test/TimerManagerTests.h | 142 + lib/cpp/src/processor/PeekProcessor.cpp | 122 + lib/cpp/src/processor/PeekProcessor.h | 77 + lib/cpp/src/processor/StatsProcessor.h | 264 ++ lib/cpp/src/protocol/TBase64Utils.cpp | 79 + lib/cpp/src/protocol/TBase64Utils.h | 42 + lib/cpp/src/protocol/TBinaryProtocol.cpp | 394 +++ lib/cpp/src/protocol/TBinaryProtocol.h | 254 ++ lib/cpp/src/protocol/TCompactProtocol.cpp | 736 ++++ lib/cpp/src/protocol/TCompactProtocol.h | 279 ++ lib/cpp/src/protocol/TDebugProtocol.cpp | 346 ++ lib/cpp/src/protocol/TDebugProtocol.h | 225 ++ lib/cpp/src/protocol/TDenseProtocol.cpp | 762 +++++ lib/cpp/src/protocol/TDenseProtocol.h | 253 ++ lib/cpp/src/protocol/TJSONProtocol.cpp | 998 ++++++ lib/cpp/src/protocol/TJSONProtocol.h | 340 ++ lib/cpp/src/protocol/TOneWayProtocol.h | 304 ++ lib/cpp/src/protocol/TProtocol.h | 438 +++ lib/cpp/src/protocol/TProtocolException.h | 104 + lib/cpp/src/protocol/TProtocolTap.h | 187 + lib/cpp/src/server/TNonblockingServer.cpp | 750 ++++ lib/cpp/src/server/TNonblockingServer.h | 434 +++ lib/cpp/src/server/TServer.cpp | 38 + lib/cpp/src/server/TServer.h | 213 ++ lib/cpp/src/server/TSimpleServer.cpp | 118 + lib/cpp/src/server/TSimpleServer.h | 70 + lib/cpp/src/server/TThreadPoolServer.cpp | 217 ++ lib/cpp/src/server/TThreadPoolServer.h | 79 + lib/cpp/src/server/TThreadedServer.cpp | 243 ++ lib/cpp/src/server/TThreadedServer.h | 74 + lib/cpp/src/transport/TBufferTransports.cpp | 370 ++ lib/cpp/src/transport/TBufferTransports.h | 667 ++++ lib/cpp/src/transport/TFDTransport.cpp | 77 + lib/cpp/src/transport/TFDTransport.h | 73 + lib/cpp/src/transport/TFileTransport.cpp | 953 ++++++ lib/cpp/src/transport/TFileTransport.h | 440 +++ lib/cpp/src/transport/THttpClient.cpp | 348 ++ lib/cpp/src/transport/THttpClient.h | 111 + lib/cpp/src/transport/TServerSocket.cpp | 366 ++ lib/cpp/src/transport/TServerSocket.h | 76 + lib/cpp/src/transport/TServerTransport.h | 92 + lib/cpp/src/transport/TShortReadTransport.h | 96 + .../src/transport/TSimpleFileTransport.cpp | 54 + lib/cpp/src/transport/TSimpleFileTransport.h | 41 + lib/cpp/src/transport/TSocket.cpp | 589 ++++ lib/cpp/src/transport/TSocket.h | 242 ++ lib/cpp/src/transport/TSocketPool.cpp | 235 ++ lib/cpp/src/transport/TSocketPool.h | 191 ++ lib/cpp/src/transport/TTransport.h | 224 ++ lib/cpp/src/transport/TTransportException.cpp | 31 + lib/cpp/src/transport/TTransportException.h | 117 + lib/cpp/src/transport/TTransportUtils.cpp | 178 + lib/cpp/src/transport/TTransportUtils.h | 287 ++ lib/cpp/src/transport/TZlibTransport.cpp | 299 ++ lib/cpp/src/transport/TZlibTransport.h | 219 ++ lib/cpp/thrift-nb.pc.in | 30 + lib/cpp/thrift-z.pc.in | 30 + lib/cpp/thrift.pc.in | 29 + lib/csharp/Makefile.am | 70 + lib/csharp/README | 26 + .../Properties/AssemblyInfo.cs | 55 + lib/csharp/ThriftMSBuildTask/ThriftBuild.cs | 242 ++ .../ThriftMSBuildTask.csproj | 66 + lib/csharp/src/Collections/THashSet.cs | 142 + lib/csharp/src/Protocol/TBase.cs | 34 + lib/csharp/src/Protocol/TBinaryProtocol.cs | 392 +++ lib/csharp/src/Protocol/TField.cs | 58 + lib/csharp/src/Protocol/TList.cs | 50 + lib/csharp/src/Protocol/TMap.cs | 58 + lib/csharp/src/Protocol/TMessage.cs | 58 + lib/csharp/src/Protocol/TMessageType.cs | 31 + lib/csharp/src/Protocol/TProtocol.cs | 87 + lib/csharp/src/Protocol/TProtocolException.cs | 61 + lib/csharp/src/Protocol/TProtocolFactory.cs | 29 + lib/csharp/src/Protocol/TProtocolUtil.cs | 94 + lib/csharp/src/Protocol/TSet.cs | 50 + lib/csharp/src/Protocol/TStruct.cs | 42 + lib/csharp/src/Protocol/TType.cs | 40 + lib/csharp/src/Server/TServer.cs | 135 + lib/csharp/src/Server/TSimpleServer.cs | 148 + lib/csharp/src/Server/TThreadPoolServer.cs | 186 + lib/csharp/src/Server/TThreadedServer.cs | 234 ++ lib/csharp/src/TApplicationException.cs | 131 + lib/csharp/src/TProcessor.cs | 29 + lib/csharp/src/Thrift.csproj | 77 + lib/csharp/src/Thrift.sln | 51 + .../src/Transport/TBufferedTransport.cs | 100 + lib/csharp/src/Transport/TServerSocket.cs | 157 + lib/csharp/src/Transport/TServerTransport.cs | 39 + lib/csharp/src/Transport/TSocket.cs | 144 + lib/csharp/src/Transport/TStreamTransport.cs | 103 + lib/csharp/src/Transport/TTransport.cs | 66 + .../src/Transport/TTransportException.cs | 64 + lib/csharp/src/Transport/TTransportFactory.cs | 38 + lib/erl/Makefile | 37 + lib/erl/README | 56 + lib/erl/build/beamver | 59 + lib/erl/build/buildtargets.mk | 15 + lib/erl/build/colors.mk | 24 + lib/erl/build/docs.mk | 12 + lib/erl/build/mime.types | 98 + lib/erl/build/otp.mk | 146 + lib/erl/build/otp_subdir.mk | 85 + lib/erl/build/raw_test.mk | 29 + lib/erl/include/thrift_constants.hrl | 54 + lib/erl/include/thrift_protocol.hrl | 31 + lib/erl/src/Makefile | 116 + lib/erl/src/test_handler.erl | 26 + lib/erl/src/test_service.erl | 29 + lib/erl/src/thrift.app.src | 44 + lib/erl/src/thrift.appup.src | 1 + lib/erl/src/thrift_base64_transport.erl | 64 + lib/erl/src/thrift_binary_protocol.erl | 325 ++ lib/erl/src/thrift_buffered_transport.erl | 180 + lib/erl/src/thrift_client.erl | 330 ++ lib/erl/src/thrift_disk_log_transport.erl | 118 + lib/erl/src/thrift_file_transport.erl | 87 + lib/erl/src/thrift_framed_transport.erl | 208 ++ lib/erl/src/thrift_http_transport.erl | 199 ++ lib/erl/src/thrift_memory_buffer.erl | 164 + lib/erl/src/thrift_processor.erl | 188 ++ lib/erl/src/thrift_protocol.erl | 356 ++ lib/erl/src/thrift_server.erl | 183 + lib/erl/src/thrift_service.erl | 25 + lib/erl/src/thrift_socket_server.erl | 249 ++ lib/erl/src/thrift_socket_transport.erl | 119 + lib/erl/src/thrift_transport.erl | 57 + lib/erl/vsn.mk | 1 + lib/hs/.gitignore | 1 + lib/hs/README | 82 + lib/hs/Setup.lhs | 23 + lib/hs/TODO | 2 + lib/hs/Thrift.cabal | 20 + lib/hs/src/Thrift.hs | 111 + lib/hs/src/Thrift/Protocol.hs | 191 ++ lib/hs/src/Thrift/Protocol/Binary.hs | 147 + lib/hs/src/Thrift/Server.hs | 65 + lib/hs/src/Thrift/Transport.hs | 60 + lib/hs/src/Thrift/Transport/Handle.hs | 58 + lib/java/Makefile.am | 36 + lib/java/README | 43 + lib/java/build.xml | 192 ++ lib/java/ivy.xml | 7 + .../src/org/apache/thrift/IntRangeSet.java | 171 + .../apache/thrift/TApplicationException.java | 122 + lib/java/src/org/apache/thrift/TBase.java | 66 + .../apache/thrift/TByteArrayOutputStream.java | 46 + .../src/org/apache/thrift/TDeserializer.java | 94 + .../src/org/apache/thrift/TException.java | 45 + .../apache/thrift/TFieldRequirementType.java | 30 + .../src/org/apache/thrift/TProcessor.java | 32 + .../org/apache/thrift/TProcessorFactory.java | 39 + .../src/org/apache/thrift/TSerializer.java | 110 + .../thrift/meta_data/FieldMetaData.java | 69 + .../thrift/meta_data/FieldValueMetaData.java | 42 + .../apache/thrift/meta_data/ListMetaData.java | 29 + .../apache/thrift/meta_data/MapMetaData.java | 31 + .../apache/thrift/meta_data/SetMetaData.java | 29 + .../thrift/meta_data/StructMetaData.java | 31 + .../apache/thrift/protocol/TBase64Utils.java | 128 + .../thrift/protocol/TBinaryProtocol.java | 331 ++ .../thrift/protocol/TCompactProtocol.java | 741 ++++ .../org/apache/thrift/protocol/TField.java | 48 + .../apache/thrift/protocol/TJSONProtocol.java | 927 +++++ .../src/org/apache/thrift/protocol/TList.java | 38 + .../src/org/apache/thrift/protocol/TMap.java | 40 + .../org/apache/thrift/protocol/TMessage.java | 48 + .../apache/thrift/protocol/TMessageType.java | 31 + .../org/apache/thrift/protocol/TProtocol.java | 146 + .../thrift/protocol/TProtocolException.java | 81 + .../thrift/protocol/TProtocolFactory.java | 30 + .../apache/thrift/protocol/TProtocolUtil.java | 158 + .../src/org/apache/thrift/protocol/TSet.java | 42 + .../thrift/protocol/TSimpleJSONProtocol.java | 384 +++ .../org/apache/thrift/protocol/TStruct.java | 36 + .../src/org/apache/thrift/protocol/TType.java | 40 + .../org/apache/thrift/server/THsHaServer.java | 304 ++ .../thrift/server/TNonblockingServer.java | 769 +++++ .../src/org/apache/thrift/server/TServer.java | 126 + .../apache/thrift/server/TSimpleServer.java | 143 + .../thrift/server/TThreadPoolServer.java | 270 ++ .../thrift/transport/TFramedTransport.java | 122 + .../apache/thrift/transport/THttpClient.java | 157 + .../thrift/transport/TIOStreamTransport.java | 158 + .../thrift/transport/TMemoryBuffer.java | 98 + .../transport/TNonblockingServerSocket.java | 160 + .../TNonblockingServerTransport.java | 31 + .../thrift/transport/TNonblockingSocket.java | 213 ++ .../transport/TNonblockingTransport.java | 31 + .../thrift/transport/TServerSocket.java | 144 + .../thrift/transport/TServerTransport.java | 51 + .../org/apache/thrift/transport/TSocket.java | 206 ++ .../apache/thrift/transport/TTransport.java | 121 + .../thrift/transport/TTransportException.java | 80 + .../thrift/transport/TTransportFactory.java | 41 + lib/java/test/TestClient | 22 + lib/java/test/TestNonblockingServer | 22 + lib/java/test/TestServer | 22 + .../org/apache/thrift/test/DeepCopyTest.java | 138 + .../org/apache/thrift/test/EqualityTest.java | 661 ++++ .../test/org/apache/thrift/test/Fixtures.java | 127 + .../org/apache/thrift/test/IdentityTest.java | 175 + .../org/apache/thrift/test/JSONProtoTest.java | 183 + .../org/apache/thrift/test/JavaBeansTest.java | 110 + .../org/apache/thrift/test/MetaDataTest.java | 77 + .../test/OverloadNonblockingServer.java | 48 + .../org/apache/thrift/test/ReadStruct.java | 62 + .../thrift/test/SerializationBenchmark.java | 87 + .../thrift/test/TCompactProtocolTest.java | 450 +++ .../org/apache/thrift/test/TestClient.java | 423 +++ .../thrift/test/TestNonblockingServer.java | 73 + .../org/apache/thrift/test/TestServer.java | 306 ++ .../org/apache/thrift/test/ToStringTest.java | 92 + .../org/apache/thrift/test/WriteStruct.java | 47 + lib/ocaml/Makefile | 23 + lib/ocaml/OCamlMakefile | 1231 +++++++ lib/ocaml/README | 119 + lib/ocaml/README-OCamlMakefile | 640 ++++ lib/ocaml/TODO | 5 + lib/ocaml/src/Makefile | 26 + lib/ocaml/src/TBinaryProtocol.ml | 171 + lib/ocaml/src/TChannelTransport.ml | 39 + lib/ocaml/src/TServer.ml | 42 + lib/ocaml/src/TServerSocket.ml | 41 + lib/ocaml/src/TSimpleServer.ml | 38 + lib/ocaml/src/TSocket.ml | 59 + lib/ocaml/src/TThreadedServer.ml | 45 + lib/ocaml/src/Thrift.ml | 368 ++ lib/perl/Makefile.PL | 29 + lib/perl/Makefile.am | 54 + lib/perl/README | 41 + lib/perl/lib/Thrift.pm | 177 + lib/perl/lib/Thrift/BinaryProtocol.pm | 498 +++ lib/perl/lib/Thrift/BufferedTransport.pm | 109 + lib/perl/lib/Thrift/FramedTransport.pm | 164 + lib/perl/lib/Thrift/HttpClient.pm | 200 ++ lib/perl/lib/Thrift/MemoryBuffer.pm | 126 + lib/perl/lib/Thrift/Protocol.pm | 543 +++ lib/perl/lib/Thrift/Socket.pm | 271 ++ lib/perl/lib/Thrift/Transport.pm | 129 + lib/perl/test.pl | 25 + lib/perl/test/Makefile.am | 31 + lib/perl/test/memory_buffer.t | 53 + lib/php/README | 63 + lib/php/README.apache | 62 + lib/php/src/Thrift.php | 787 +++++ lib/php/src/autoload.php | 51 + lib/php/src/ext/thrift_protocol/config.m4 | 13 + .../thrift_protocol/php_thrift_protocol.cpp | 999 ++++++ .../ext/thrift_protocol/php_thrift_protocol.h | 26 + lib/php/src/protocol/TBinaryProtocol.php | 431 +++ lib/php/src/protocol/TProtocol.php | 377 +++ lib/php/src/transport/TBufferedTransport.php | 163 + lib/php/src/transport/TFramedTransport.php | 179 + lib/php/src/transport/THttpClient.php | 202 ++ lib/php/src/transport/TMemoryBuffer.php | 84 + lib/php/src/transport/TNullTransport.php | 48 + lib/php/src/transport/TPhpStream.php | 111 + lib/php/src/transport/TSocket.php | 312 ++ lib/php/src/transport/TSocketPool.php | 296 ++ lib/php/src/transport/TTransport.php | 108 + lib/py/Makefile.am | 36 + lib/py/README | 35 + lib/py/setup.py | 43 + lib/py/src/TSCons.py | 33 + lib/py/src/Thrift.py | 123 + lib/py/src/__init__.py | 20 + lib/py/src/protocol/TBinaryProtocol.py | 259 ++ lib/py/src/protocol/TProtocol.py | 205 ++ lib/py/src/protocol/__init__.py | 20 + lib/py/src/protocol/fastbinary.c | 1203 +++++++ lib/py/src/server/THttpServer.py | 63 + lib/py/src/server/TNonblockingServer.py | 309 ++ lib/py/src/server/TServer.py | 270 ++ lib/py/src/server/__init__.py | 20 + lib/py/src/transport/THttpClient.py | 100 + lib/py/src/transport/TSocket.py | 147 + lib/py/src/transport/TTransport.py | 326 ++ lib/py/src/transport/TTwisted.py | 177 + lib/py/src/transport/__init__.py | 20 + lib/rb/CHANGELOG | 1 + lib/rb/Makefile.am | 47 + lib/rb/Manifest | 81 + lib/rb/README | 43 + lib/rb/Rakefile | 103 + lib/rb/benchmark/Benchmark.thrift | 24 + lib/rb/benchmark/benchmark.rb | 271 ++ lib/rb/benchmark/client.rb | 74 + lib/rb/benchmark/server.rb | 82 + lib/rb/benchmark/thin_server.rb | 44 + lib/rb/ext/binary_protocol_accelerated.c | 474 +++ lib/rb/ext/binary_protocol_accelerated.h | 20 + lib/rb/ext/compact_protocol.c | 665 ++++ lib/rb/ext/compact_protocol.h | 20 + lib/rb/ext/constants.h | 95 + lib/rb/ext/extconf.rb | 26 + lib/rb/ext/macros.h | 41 + lib/rb/ext/memory_buffer.c | 72 + lib/rb/ext/memory_buffer.h | 20 + lib/rb/ext/protocol.c | 185 + lib/rb/ext/protocol.h | 20 + lib/rb/ext/struct.c | 605 ++++ lib/rb/ext/struct.h | 67 + lib/rb/ext/thrift_native.c | 194 ++ lib/rb/lib/thrift.rb | 59 + lib/rb/lib/thrift/client.rb | 62 + lib/rb/lib/thrift/core_ext.rb | 23 + lib/rb/lib/thrift/core_ext/fixnum.rb | 29 + lib/rb/lib/thrift/exceptions.rb | 82 + lib/rb/lib/thrift/processor.rb | 57 + lib/rb/lib/thrift/protocol/base_protocol.rb | 290 ++ lib/rb/lib/thrift/protocol/binary_protocol.rb | 225 ++ .../protocol/binary_protocol_accelerated.rb | 35 + .../lib/thrift/protocol/compact_protocol.rb | 422 +++ lib/rb/lib/thrift/serializer/deserializer.rb | 33 + lib/rb/lib/thrift/serializer/serializer.rb | 34 + lib/rb/lib/thrift/server/base_server.rb | 31 + .../lib/thrift/server/mongrel_http_server.rb | 58 + .../lib/thrift/server/nonblocking_server.rb | 296 ++ lib/rb/lib/thrift/server/simple_server.rb | 43 + .../lib/thrift/server/thread_pool_server.rb | 75 + lib/rb/lib/thrift/server/threaded_server.rb | 47 + lib/rb/lib/thrift/struct.rb | 294 ++ lib/rb/lib/thrift/thrift_native.rb | 24 + .../thrift/transport/base_server_transport.rb | 37 + lib/rb/lib/thrift/transport/base_transport.rb | 70 + .../thrift/transport/buffered_transport.rb | 77 + .../lib/thrift/transport/framed_transport.rb | 90 + .../thrift/transport/http_client_transport.rb | 45 + .../thrift/transport/io_stream_transport.rb | 39 + .../transport/memory_buffer_transport.rb | 93 + lib/rb/lib/thrift/transport/server_socket.rb | 63 + lib/rb/lib/thrift/transport/socket.rb | 136 + .../thrift/transport/unix_server_socket.rb | 60 + lib/rb/lib/thrift/transport/unix_socket.rb | 40 + lib/rb/lib/thrift/types.rb | 101 + lib/rb/script/proto_benchmark.rb | 121 + lib/rb/script/read_struct.rb | 43 + lib/rb/script/write_struct.rb | 30 + lib/rb/setup.rb | 1585 +++++++++ lib/rb/spec/ThriftSpec.thrift | 84 + lib/rb/spec/base_protocol_spec.rb | 160 + lib/rb/spec/base_transport_spec.rb | 344 ++ .../spec/binary_protocol_accelerated_spec.rb | 42 + lib/rb/spec/binary_protocol_spec.rb | 63 + lib/rb/spec/binary_protocol_spec_shared.rb | 375 ++ lib/rb/spec/client_spec.rb | 100 + lib/rb/spec/compact_protocol_spec.rb | 117 + lib/rb/spec/exception_spec.rb | 142 + lib/rb/spec/http_client_spec.rb | 49 + lib/rb/spec/mongrel_http_server_spec.rb | 117 + lib/rb/spec/nonblocking_server_spec.rb | 266 ++ lib/rb/spec/processor_spec.rb | 83 + lib/rb/spec/serializer_spec.rb | 70 + lib/rb/spec/server_socket_spec.rb | 80 + lib/rb/spec/server_spec.rb | 160 + lib/rb/spec/socket_spec.rb | 61 + lib/rb/spec/socket_spec_shared.rb | 104 + lib/rb/spec/spec_helper.rb | 55 + lib/rb/spec/struct_spec.rb | 253 ++ lib/rb/spec/types_spec.rb | 117 + lib/rb/spec/unix_socket_spec.rb | 108 + lib/st/README | 35 + lib/st/thrift.st | 812 +++++ print_version.sh | 47 + test/AllProtocolTests.cpp | 42 + test/AllProtocolTests.tcc | 227 ++ test/AnnotationTest.thrift | 31 + test/Benchmark.cpp | 108 + test/BrokenConstants.thrift | 25 + test/ConstantsDemo.thrift | 69 + test/DebugProtoTest.cpp | 105 + test/DebugProtoTest.thrift | 253 ++ test/DebugProtoTest_extras.cpp | 32 + test/DenseLinkingTest.thrift | 92 + test/DenseProtoTest.cpp | 384 +++ test/DocTest.thrift | 247 ++ test/FastbinaryTest.py | 222 ++ test/GenericHelpers.h | 102 + test/JSONProtoTest.cpp | 159 + test/JavaBeansTest.thrift | 34 + test/Makefile.am | 178 + test/ManyTypedefs.thrift | 50 + test/OptionalRequiredTest.cpp | 242 ++ test/OptionalRequiredTest.thrift | 72 + test/SmallTest.thrift | 60 + test/StressTest.thrift | 33 + test/TBufferBaseTest.cpp | 639 ++++ test/TFDTransportTest.cpp | 56 + test/TMemoryBufferTest.cpp | 107 + test/TPipedTransportTest.cpp | 54 + test/ThriftTest.thrift | 172 + test/ThriftTest_extras.cpp | 33 + test/UnitTestMain.cpp | 21 + test/ZlibTest.cpp | 310 ++ test/cpp/Stress-test.mk | 71 + test/cpp/Thrift-test.mk | 78 + test/cpp/realloc/Makefile | 40 + test/cpp/realloc/realloc_test.c | 107 + test/cpp/src/TestClient.cpp | 497 +++ test/cpp/src/TestServer.cpp | 424 +++ test/cpp/src/main.cpp | 509 +++ test/cpp/src/nb-main.cpp | 502 +++ test/csharp/CSharpClient.cs | 87 + test/csharp/CSharpServer.cs | 129 + test/csharp/ThriftTest/Program.cs | 61 + .../ThriftTest/Properties/AssemblyInfo.cs | 55 + test/csharp/ThriftTest/TestClient.cs | 425 +++ test/csharp/ThriftTest/TestServer.cs | 348 ++ test/csharp/ThriftTest/ThriftTest.csproj | 111 + test/csharp/ThriftTest/maketest.sh | 23 + test/erl/Makefile | 66 + test/erl/src/stress_server.erl | 64 + test/erl/src/test_disklog.erl | 81 + test/erl/src/test_membuffer.erl | 81 + test/erl/src/test_server.erl | 174 + test/hs/Client.hs | 58 + test/hs/Server.hs | 57 + test/hs/runclient.sh | 26 + test/hs/runserver.sh | 27 + test/ocaml/Makefile | 24 + test/ocaml/client/Makefile | 26 + test/ocaml/client/TestClient.ml | 82 + test/ocaml/server/Makefile | 27 + test/ocaml/server/TestServer.ml | 136 + test/perl/Makefile | 30 + test/perl/TestClient.pl | 338 ++ test/php/Makefile | 39 + test/php/TestClient.php | 398 +++ test/php/TestInline.php | 24 + test/py/Makefile.am | 48 + test/py/RunClientServer.py | 60 + test/py/SerializationTest.py | 133 + test/py/TestClient.py | 156 + test/py/TestEof.py | 121 + test/py/TestServer.py | 115 + test/py/TestSocket.py | 84 + test/py/TestSyntax.py | 28 + test/py/explicit_module/runtest.sh | 34 + test/py/explicit_module/test1.thrift | 24 + test/py/explicit_module/test2.thrift | 24 + test/rb/Makefile.am | 28 + test/rb/benchmarks/protocol_benchmark.rb | 174 + test/rb/core/test_backwards_compatability.rb | 30 + test/rb/core/test_exceptions.rb | 30 + test/rb/core/transport/test_transport.rb | 70 + test/rb/fixtures/structs.rb | 298 ++ test/rb/generation/test_enum.rb | 28 + test/rb/generation/test_struct.rb | 48 + .../accelerated_buffered_client.rb | 163 + .../accelerated_buffered_server.rb | 65 + test/rb/integration/buffered_client.rb | 163 + test/rb/integration/simple_client.rb | 163 + test/rb/integration/simple_server.rb | 64 + test/rb/integration/test_simple_handler.rb | 211 ++ test/rb/test_helper.rb | 35 + test/rb/test_suite.rb | 20 + test/threads/Makefile | 63 + test/threads/ThreadsClient.cpp | 63 + test/threads/ThreadsServer.cpp | 141 + test/threads/ThreadsTest.thrift | 28 + tutorial/README | 42 + tutorial/cpp/CppClient.cpp | 84 + tutorial/cpp/CppServer.cpp | 154 + tutorial/cpp/Makefile | 35 + tutorial/erl/client.erl | 74 + tutorial/erl/client.sh | 1 + tutorial/erl/server.erl | 82 + tutorial/erl/server.sh | 37 + tutorial/java/JavaClient | 22 + tutorial/java/JavaServer | 22 + tutorial/java/build.xml | 47 + tutorial/java/src/JavaClient.java | 85 + tutorial/java/src/JavaServer.java | 119 + tutorial/perl/PerlClient.pl | 82 + tutorial/php/PhpClient.php | 92 + tutorial/php/PhpServer.php | 132 + tutorial/php/runserver.py | 32 + tutorial/py/PythonClient.py | 82 + tutorial/py/PythonServer.py | 96 + tutorial/rb/RubyClient.rb | 75 + tutorial/rb/RubyServer.rb | 95 + tutorial/shared.thrift | 36 + tutorial/tutorial.thrift | 152 + 631 files changed, 114437 insertions(+) create mode 100644 .gitignore create mode 100644 CHANGES create mode 100644 CONTRIBUTORS create mode 100644 DISCLAIMER create mode 100644 LICENSE create mode 100644 Makefile.am create mode 100644 NEWS create mode 100644 NOTICE create mode 100644 README create mode 100644 aclocal/ax_boost_base.m4 create mode 100644 aclocal/ax_javac_and_java.m4 create mode 100644 aclocal/ax_lib_event.m4 create mode 100644 aclocal/ax_lib_zlib.m4 create mode 100644 aclocal/ax_signed_right_shift.m4 create mode 100644 aclocal/ax_thrift_internal.m4 create mode 100755 bootstrap.sh create mode 100755 cleanup.sh create mode 100644 compiler/cpp/Makefile.am create mode 100644 compiler/cpp/README create mode 100644 compiler/cpp/src/generate/t_cocoa_generator.cc create mode 100644 compiler/cpp/src/generate/t_cpp_generator.cc create mode 100644 compiler/cpp/src/generate/t_csharp_generator.cc create mode 100644 compiler/cpp/src/generate/t_erl_generator.cc create mode 100644 compiler/cpp/src/generate/t_generator.cc create mode 100644 compiler/cpp/src/generate/t_generator.h create mode 100644 compiler/cpp/src/generate/t_hs_generator.cc create mode 100644 compiler/cpp/src/generate/t_html_generator.cc create mode 100644 compiler/cpp/src/generate/t_java_generator.cc create mode 100644 compiler/cpp/src/generate/t_ocaml_generator.cc create mode 100644 compiler/cpp/src/generate/t_oop_generator.h create mode 100644 compiler/cpp/src/generate/t_perl_generator.cc create mode 100644 compiler/cpp/src/generate/t_php_generator.cc create mode 100644 compiler/cpp/src/generate/t_py_generator.cc create mode 100644 compiler/cpp/src/generate/t_rb_generator.cc create mode 100644 compiler/cpp/src/generate/t_st_generator.cc create mode 100644 compiler/cpp/src/generate/t_xsd_generator.cc create mode 100644 compiler/cpp/src/globals.h create mode 100644 compiler/cpp/src/main.cc create mode 100644 compiler/cpp/src/main.h create mode 100644 compiler/cpp/src/md5.c create mode 100644 compiler/cpp/src/md5.h create mode 100644 compiler/cpp/src/parse/t_base_type.h create mode 100644 compiler/cpp/src/parse/t_const.h create mode 100644 compiler/cpp/src/parse/t_const_value.h create mode 100644 compiler/cpp/src/parse/t_container.h create mode 100644 compiler/cpp/src/parse/t_doc.h create mode 100644 compiler/cpp/src/parse/t_enum.h create mode 100644 compiler/cpp/src/parse/t_enum_value.h create mode 100644 compiler/cpp/src/parse/t_field.h create mode 100644 compiler/cpp/src/parse/t_function.h create mode 100644 compiler/cpp/src/parse/t_list.h create mode 100644 compiler/cpp/src/parse/t_map.h create mode 100644 compiler/cpp/src/parse/t_program.h create mode 100644 compiler/cpp/src/parse/t_scope.h create mode 100644 compiler/cpp/src/parse/t_service.h create mode 100644 compiler/cpp/src/parse/t_set.h create mode 100644 compiler/cpp/src/parse/t_struct.h create mode 100644 compiler/cpp/src/parse/t_type.h create mode 100644 compiler/cpp/src/parse/t_typedef.h create mode 100644 compiler/cpp/src/platform.h create mode 100644 compiler/cpp/src/thriftl.ll create mode 100644 compiler/cpp/src/thrifty.yy create mode 100644 configure.ac create mode 100644 contrib/fb303/LICENSE create mode 100644 contrib/fb303/Makefile.am create mode 100644 contrib/fb303/README create mode 100644 contrib/fb303/acinclude.m4 create mode 100644 contrib/fb303/aclocal/ax_boost_base.m4 create mode 100755 contrib/fb303/bootstrap.sh create mode 100644 contrib/fb303/configure.ac create mode 100644 contrib/fb303/cpp/FacebookBase.cpp create mode 100644 contrib/fb303/cpp/FacebookBase.h create mode 100644 contrib/fb303/cpp/Makefile.am create mode 100644 contrib/fb303/cpp/ServiceTracker.cpp create mode 100644 contrib/fb303/cpp/ServiceTracker.h create mode 100644 contrib/fb303/global_footer.mk create mode 100644 contrib/fb303/global_header.mk create mode 100644 contrib/fb303/if/fb303.thrift create mode 100644 contrib/fb303/java/FacebookBase.java create mode 100755 contrib/fb303/java/build.xml create mode 100644 contrib/fb303/php/FacebookBase.php create mode 100644 contrib/fb303/py/Makefile.am create mode 100644 contrib/fb303/py/fb303/FacebookBase.py create mode 100644 contrib/fb303/py/fb303_scripts/__init__.py create mode 100644 contrib/fb303/py/fb303_scripts/fb303_simple_mgmt.py create mode 100644 contrib/fb303/py/setup.py create mode 100644 contrib/thrift.el create mode 100644 contrib/thrift.spec create mode 100644 contrib/thrift.vim create mode 100644 contrib/thrift_dump.cpp create mode 100644 doc/lgpl-2.1.txt create mode 100644 doc/otp-base-license.txt create mode 100644 doc/thrift.bnf create mode 100644 doc/thrift.tex create mode 100644 lib/Makefile.am create mode 100644 lib/cocoa/README create mode 100644 lib/cocoa/src/TApplicationException.h create mode 100644 lib/cocoa/src/TApplicationException.m create mode 100644 lib/cocoa/src/TException.h create mode 100644 lib/cocoa/src/TException.m create mode 100644 lib/cocoa/src/TProcessor.h create mode 100644 lib/cocoa/src/protocol/TBinaryProtocol.h create mode 100644 lib/cocoa/src/protocol/TBinaryProtocol.m create mode 100644 lib/cocoa/src/protocol/TProtocol.h create mode 100644 lib/cocoa/src/protocol/TProtocolException.h create mode 100644 lib/cocoa/src/protocol/TProtocolException.m create mode 100644 lib/cocoa/src/protocol/TProtocolFactory.h create mode 100644 lib/cocoa/src/protocol/TProtocolUtil.h create mode 100644 lib/cocoa/src/protocol/TProtocolUtil.m create mode 100644 lib/cocoa/src/server/TSocketServer.h create mode 100644 lib/cocoa/src/server/TSocketServer.m create mode 100644 lib/cocoa/src/transport/THTTPClient.h create mode 100644 lib/cocoa/src/transport/THTTPClient.m create mode 100644 lib/cocoa/src/transport/TNSFileHandleTransport.h create mode 100644 lib/cocoa/src/transport/TNSFileHandleTransport.m create mode 100644 lib/cocoa/src/transport/TNSStreamTransport.h create mode 100644 lib/cocoa/src/transport/TNSStreamTransport.m create mode 100644 lib/cocoa/src/transport/TSocketClient.h create mode 100644 lib/cocoa/src/transport/TSocketClient.m create mode 100644 lib/cocoa/src/transport/TTransport.h create mode 100644 lib/cocoa/src/transport/TTransportException.h create mode 100644 lib/cocoa/src/transport/TTransportException.m create mode 100644 lib/cpp/Makefile.am create mode 100644 lib/cpp/README create mode 100644 lib/cpp/src/TLogging.h create mode 100644 lib/cpp/src/TProcessor.h create mode 100644 lib/cpp/src/TReflectionLocal.h create mode 100644 lib/cpp/src/Thrift.cpp create mode 100644 lib/cpp/src/Thrift.h create mode 100644 lib/cpp/src/concurrency/Exception.h create mode 100644 lib/cpp/src/concurrency/FunctionRunner.h create mode 100644 lib/cpp/src/concurrency/Monitor.cpp create mode 100644 lib/cpp/src/concurrency/Monitor.h create mode 100644 lib/cpp/src/concurrency/Mutex.cpp create mode 100644 lib/cpp/src/concurrency/Mutex.h create mode 100644 lib/cpp/src/concurrency/PosixThreadFactory.cpp create mode 100644 lib/cpp/src/concurrency/PosixThreadFactory.h create mode 100644 lib/cpp/src/concurrency/Thread.h create mode 100644 lib/cpp/src/concurrency/ThreadManager.cpp create mode 100644 lib/cpp/src/concurrency/ThreadManager.h create mode 100644 lib/cpp/src/concurrency/TimerManager.cpp create mode 100644 lib/cpp/src/concurrency/TimerManager.h create mode 100644 lib/cpp/src/concurrency/Util.cpp create mode 100644 lib/cpp/src/concurrency/Util.h create mode 100644 lib/cpp/src/concurrency/test/Tests.cpp create mode 100644 lib/cpp/src/concurrency/test/ThreadFactoryTests.h create mode 100644 lib/cpp/src/concurrency/test/ThreadManagerTests.h create mode 100644 lib/cpp/src/concurrency/test/TimerManagerTests.h create mode 100644 lib/cpp/src/processor/PeekProcessor.cpp create mode 100644 lib/cpp/src/processor/PeekProcessor.h create mode 100644 lib/cpp/src/processor/StatsProcessor.h create mode 100644 lib/cpp/src/protocol/TBase64Utils.cpp create mode 100644 lib/cpp/src/protocol/TBase64Utils.h create mode 100644 lib/cpp/src/protocol/TBinaryProtocol.cpp create mode 100644 lib/cpp/src/protocol/TBinaryProtocol.h create mode 100644 lib/cpp/src/protocol/TCompactProtocol.cpp create mode 100644 lib/cpp/src/protocol/TCompactProtocol.h create mode 100644 lib/cpp/src/protocol/TDebugProtocol.cpp create mode 100644 lib/cpp/src/protocol/TDebugProtocol.h create mode 100644 lib/cpp/src/protocol/TDenseProtocol.cpp create mode 100644 lib/cpp/src/protocol/TDenseProtocol.h create mode 100644 lib/cpp/src/protocol/TJSONProtocol.cpp create mode 100644 lib/cpp/src/protocol/TJSONProtocol.h create mode 100644 lib/cpp/src/protocol/TOneWayProtocol.h create mode 100644 lib/cpp/src/protocol/TProtocol.h create mode 100644 lib/cpp/src/protocol/TProtocolException.h create mode 100644 lib/cpp/src/protocol/TProtocolTap.h create mode 100644 lib/cpp/src/server/TNonblockingServer.cpp create mode 100644 lib/cpp/src/server/TNonblockingServer.h create mode 100644 lib/cpp/src/server/TServer.cpp create mode 100644 lib/cpp/src/server/TServer.h create mode 100644 lib/cpp/src/server/TSimpleServer.cpp create mode 100644 lib/cpp/src/server/TSimpleServer.h create mode 100644 lib/cpp/src/server/TThreadPoolServer.cpp create mode 100644 lib/cpp/src/server/TThreadPoolServer.h create mode 100644 lib/cpp/src/server/TThreadedServer.cpp create mode 100644 lib/cpp/src/server/TThreadedServer.h create mode 100644 lib/cpp/src/transport/TBufferTransports.cpp create mode 100644 lib/cpp/src/transport/TBufferTransports.h create mode 100644 lib/cpp/src/transport/TFDTransport.cpp create mode 100644 lib/cpp/src/transport/TFDTransport.h create mode 100644 lib/cpp/src/transport/TFileTransport.cpp create mode 100644 lib/cpp/src/transport/TFileTransport.h create mode 100644 lib/cpp/src/transport/THttpClient.cpp create mode 100644 lib/cpp/src/transport/THttpClient.h create mode 100644 lib/cpp/src/transport/TServerSocket.cpp create mode 100644 lib/cpp/src/transport/TServerSocket.h create mode 100644 lib/cpp/src/transport/TServerTransport.h create mode 100644 lib/cpp/src/transport/TShortReadTransport.h create mode 100644 lib/cpp/src/transport/TSimpleFileTransport.cpp create mode 100644 lib/cpp/src/transport/TSimpleFileTransport.h create mode 100644 lib/cpp/src/transport/TSocket.cpp create mode 100644 lib/cpp/src/transport/TSocket.h create mode 100644 lib/cpp/src/transport/TSocketPool.cpp create mode 100644 lib/cpp/src/transport/TSocketPool.h create mode 100644 lib/cpp/src/transport/TTransport.h create mode 100644 lib/cpp/src/transport/TTransportException.cpp create mode 100644 lib/cpp/src/transport/TTransportException.h create mode 100644 lib/cpp/src/transport/TTransportUtils.cpp create mode 100644 lib/cpp/src/transport/TTransportUtils.h create mode 100644 lib/cpp/src/transport/TZlibTransport.cpp create mode 100644 lib/cpp/src/transport/TZlibTransport.h create mode 100644 lib/cpp/thrift-nb.pc.in create mode 100644 lib/cpp/thrift-z.pc.in create mode 100644 lib/cpp/thrift.pc.in create mode 100644 lib/csharp/Makefile.am create mode 100644 lib/csharp/README create mode 100644 lib/csharp/ThriftMSBuildTask/Properties/AssemblyInfo.cs create mode 100644 lib/csharp/ThriftMSBuildTask/ThriftBuild.cs create mode 100644 lib/csharp/ThriftMSBuildTask/ThriftMSBuildTask.csproj create mode 100644 lib/csharp/src/Collections/THashSet.cs create mode 100644 lib/csharp/src/Protocol/TBase.cs create mode 100644 lib/csharp/src/Protocol/TBinaryProtocol.cs create mode 100644 lib/csharp/src/Protocol/TField.cs create mode 100644 lib/csharp/src/Protocol/TList.cs create mode 100644 lib/csharp/src/Protocol/TMap.cs create mode 100644 lib/csharp/src/Protocol/TMessage.cs create mode 100644 lib/csharp/src/Protocol/TMessageType.cs create mode 100644 lib/csharp/src/Protocol/TProtocol.cs create mode 100644 lib/csharp/src/Protocol/TProtocolException.cs create mode 100644 lib/csharp/src/Protocol/TProtocolFactory.cs create mode 100644 lib/csharp/src/Protocol/TProtocolUtil.cs create mode 100644 lib/csharp/src/Protocol/TSet.cs create mode 100644 lib/csharp/src/Protocol/TStruct.cs create mode 100644 lib/csharp/src/Protocol/TType.cs create mode 100644 lib/csharp/src/Server/TServer.cs create mode 100644 lib/csharp/src/Server/TSimpleServer.cs create mode 100644 lib/csharp/src/Server/TThreadPoolServer.cs create mode 100644 lib/csharp/src/Server/TThreadedServer.cs create mode 100644 lib/csharp/src/TApplicationException.cs create mode 100644 lib/csharp/src/TProcessor.cs create mode 100644 lib/csharp/src/Thrift.csproj create mode 100644 lib/csharp/src/Thrift.sln create mode 100644 lib/csharp/src/Transport/TBufferedTransport.cs create mode 100644 lib/csharp/src/Transport/TServerSocket.cs create mode 100644 lib/csharp/src/Transport/TServerTransport.cs create mode 100644 lib/csharp/src/Transport/TSocket.cs create mode 100644 lib/csharp/src/Transport/TStreamTransport.cs create mode 100644 lib/csharp/src/Transport/TTransport.cs create mode 100644 lib/csharp/src/Transport/TTransportException.cs create mode 100644 lib/csharp/src/Transport/TTransportFactory.cs create mode 100644 lib/erl/Makefile create mode 100644 lib/erl/README create mode 100644 lib/erl/build/beamver create mode 100644 lib/erl/build/buildtargets.mk create mode 100644 lib/erl/build/colors.mk create mode 100644 lib/erl/build/docs.mk create mode 100644 lib/erl/build/mime.types create mode 100644 lib/erl/build/otp.mk create mode 100644 lib/erl/build/otp_subdir.mk create mode 100644 lib/erl/build/raw_test.mk create mode 100644 lib/erl/include/thrift_constants.hrl create mode 100644 lib/erl/include/thrift_protocol.hrl create mode 100644 lib/erl/src/Makefile create mode 100644 lib/erl/src/test_handler.erl create mode 100644 lib/erl/src/test_service.erl create mode 100644 lib/erl/src/thrift.app.src create mode 100644 lib/erl/src/thrift.appup.src create mode 100644 lib/erl/src/thrift_base64_transport.erl create mode 100644 lib/erl/src/thrift_binary_protocol.erl create mode 100644 lib/erl/src/thrift_buffered_transport.erl create mode 100644 lib/erl/src/thrift_client.erl create mode 100644 lib/erl/src/thrift_disk_log_transport.erl create mode 100644 lib/erl/src/thrift_file_transport.erl create mode 100644 lib/erl/src/thrift_framed_transport.erl create mode 100644 lib/erl/src/thrift_http_transport.erl create mode 100644 lib/erl/src/thrift_memory_buffer.erl create mode 100644 lib/erl/src/thrift_processor.erl create mode 100644 lib/erl/src/thrift_protocol.erl create mode 100644 lib/erl/src/thrift_server.erl create mode 100644 lib/erl/src/thrift_service.erl create mode 100644 lib/erl/src/thrift_socket_server.erl create mode 100644 lib/erl/src/thrift_socket_transport.erl create mode 100644 lib/erl/src/thrift_transport.erl create mode 100644 lib/erl/vsn.mk create mode 100644 lib/hs/.gitignore create mode 100644 lib/hs/README create mode 100644 lib/hs/Setup.lhs create mode 100644 lib/hs/TODO create mode 100644 lib/hs/Thrift.cabal create mode 100644 lib/hs/src/Thrift.hs create mode 100644 lib/hs/src/Thrift/Protocol.hs create mode 100644 lib/hs/src/Thrift/Protocol/Binary.hs create mode 100644 lib/hs/src/Thrift/Server.hs create mode 100644 lib/hs/src/Thrift/Transport.hs create mode 100644 lib/hs/src/Thrift/Transport/Handle.hs create mode 100644 lib/java/Makefile.am create mode 100644 lib/java/README create mode 100644 lib/java/build.xml create mode 100644 lib/java/ivy.xml create mode 100644 lib/java/src/org/apache/thrift/IntRangeSet.java create mode 100644 lib/java/src/org/apache/thrift/TApplicationException.java create mode 100644 lib/java/src/org/apache/thrift/TBase.java create mode 100644 lib/java/src/org/apache/thrift/TByteArrayOutputStream.java create mode 100644 lib/java/src/org/apache/thrift/TDeserializer.java create mode 100644 lib/java/src/org/apache/thrift/TException.java create mode 100644 lib/java/src/org/apache/thrift/TFieldRequirementType.java create mode 100644 lib/java/src/org/apache/thrift/TProcessor.java create mode 100644 lib/java/src/org/apache/thrift/TProcessorFactory.java create mode 100644 lib/java/src/org/apache/thrift/TSerializer.java create mode 100644 lib/java/src/org/apache/thrift/meta_data/FieldMetaData.java create mode 100644 lib/java/src/org/apache/thrift/meta_data/FieldValueMetaData.java create mode 100644 lib/java/src/org/apache/thrift/meta_data/ListMetaData.java create mode 100644 lib/java/src/org/apache/thrift/meta_data/MapMetaData.java create mode 100644 lib/java/src/org/apache/thrift/meta_data/SetMetaData.java create mode 100644 lib/java/src/org/apache/thrift/meta_data/StructMetaData.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TBase64Utils.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TBinaryProtocol.java create mode 100755 lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TField.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TJSONProtocol.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TList.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TMap.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TMessage.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TMessageType.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TProtocol.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TProtocolException.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TProtocolFactory.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TProtocolUtil.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TSet.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TSimpleJSONProtocol.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TStruct.java create mode 100644 lib/java/src/org/apache/thrift/protocol/TType.java create mode 100644 lib/java/src/org/apache/thrift/server/THsHaServer.java create mode 100644 lib/java/src/org/apache/thrift/server/TNonblockingServer.java create mode 100644 lib/java/src/org/apache/thrift/server/TServer.java create mode 100644 lib/java/src/org/apache/thrift/server/TSimpleServer.java create mode 100644 lib/java/src/org/apache/thrift/server/TThreadPoolServer.java create mode 100644 lib/java/src/org/apache/thrift/transport/TFramedTransport.java create mode 100644 lib/java/src/org/apache/thrift/transport/THttpClient.java create mode 100644 lib/java/src/org/apache/thrift/transport/TIOStreamTransport.java create mode 100644 lib/java/src/org/apache/thrift/transport/TMemoryBuffer.java create mode 100644 lib/java/src/org/apache/thrift/transport/TNonblockingServerSocket.java create mode 100644 lib/java/src/org/apache/thrift/transport/TNonblockingServerTransport.java create mode 100644 lib/java/src/org/apache/thrift/transport/TNonblockingSocket.java create mode 100644 lib/java/src/org/apache/thrift/transport/TNonblockingTransport.java create mode 100644 lib/java/src/org/apache/thrift/transport/TServerSocket.java create mode 100644 lib/java/src/org/apache/thrift/transport/TServerTransport.java create mode 100644 lib/java/src/org/apache/thrift/transport/TSocket.java create mode 100644 lib/java/src/org/apache/thrift/transport/TTransport.java create mode 100644 lib/java/src/org/apache/thrift/transport/TTransportException.java create mode 100644 lib/java/src/org/apache/thrift/transport/TTransportFactory.java create mode 100755 lib/java/test/TestClient create mode 100644 lib/java/test/TestNonblockingServer create mode 100755 lib/java/test/TestServer create mode 100644 lib/java/test/org/apache/thrift/test/DeepCopyTest.java create mode 100644 lib/java/test/org/apache/thrift/test/EqualityTest.java create mode 100644 lib/java/test/org/apache/thrift/test/Fixtures.java create mode 100644 lib/java/test/org/apache/thrift/test/IdentityTest.java create mode 100644 lib/java/test/org/apache/thrift/test/JSONProtoTest.java create mode 100644 lib/java/test/org/apache/thrift/test/JavaBeansTest.java create mode 100644 lib/java/test/org/apache/thrift/test/MetaDataTest.java create mode 100644 lib/java/test/org/apache/thrift/test/OverloadNonblockingServer.java create mode 100644 lib/java/test/org/apache/thrift/test/ReadStruct.java create mode 100644 lib/java/test/org/apache/thrift/test/SerializationBenchmark.java create mode 100755 lib/java/test/org/apache/thrift/test/TCompactProtocolTest.java create mode 100644 lib/java/test/org/apache/thrift/test/TestClient.java create mode 100644 lib/java/test/org/apache/thrift/test/TestNonblockingServer.java create mode 100644 lib/java/test/org/apache/thrift/test/TestServer.java create mode 100644 lib/java/test/org/apache/thrift/test/ToStringTest.java create mode 100644 lib/java/test/org/apache/thrift/test/WriteStruct.java create mode 100644 lib/ocaml/Makefile create mode 100644 lib/ocaml/OCamlMakefile create mode 100644 lib/ocaml/README create mode 100644 lib/ocaml/README-OCamlMakefile create mode 100644 lib/ocaml/TODO create mode 100644 lib/ocaml/src/Makefile create mode 100644 lib/ocaml/src/TBinaryProtocol.ml create mode 100644 lib/ocaml/src/TChannelTransport.ml create mode 100644 lib/ocaml/src/TServer.ml create mode 100644 lib/ocaml/src/TServerSocket.ml create mode 100644 lib/ocaml/src/TSimpleServer.ml create mode 100644 lib/ocaml/src/TSocket.ml create mode 100644 lib/ocaml/src/TThreadedServer.ml create mode 100644 lib/ocaml/src/Thrift.ml create mode 100644 lib/perl/Makefile.PL create mode 100644 lib/perl/Makefile.am create mode 100644 lib/perl/README create mode 100644 lib/perl/lib/Thrift.pm create mode 100644 lib/perl/lib/Thrift/BinaryProtocol.pm create mode 100644 lib/perl/lib/Thrift/BufferedTransport.pm create mode 100644 lib/perl/lib/Thrift/FramedTransport.pm create mode 100644 lib/perl/lib/Thrift/HttpClient.pm create mode 100644 lib/perl/lib/Thrift/MemoryBuffer.pm create mode 100644 lib/perl/lib/Thrift/Protocol.pm create mode 100644 lib/perl/lib/Thrift/Socket.pm create mode 100644 lib/perl/lib/Thrift/Transport.pm create mode 100644 lib/perl/test.pl create mode 100644 lib/perl/test/Makefile.am create mode 100644 lib/perl/test/memory_buffer.t create mode 100644 lib/php/README create mode 100644 lib/php/README.apache create mode 100644 lib/php/src/Thrift.php create mode 100644 lib/php/src/autoload.php create mode 100644 lib/php/src/ext/thrift_protocol/config.m4 create mode 100644 lib/php/src/ext/thrift_protocol/php_thrift_protocol.cpp create mode 100644 lib/php/src/ext/thrift_protocol/php_thrift_protocol.h create mode 100644 lib/php/src/protocol/TBinaryProtocol.php create mode 100644 lib/php/src/protocol/TProtocol.php create mode 100644 lib/php/src/transport/TBufferedTransport.php create mode 100644 lib/php/src/transport/TFramedTransport.php create mode 100644 lib/php/src/transport/THttpClient.php create mode 100644 lib/php/src/transport/TMemoryBuffer.php create mode 100644 lib/php/src/transport/TNullTransport.php create mode 100644 lib/php/src/transport/TPhpStream.php create mode 100644 lib/php/src/transport/TSocket.php create mode 100644 lib/php/src/transport/TSocketPool.php create mode 100644 lib/php/src/transport/TTransport.php create mode 100644 lib/py/Makefile.am create mode 100644 lib/py/README create mode 100644 lib/py/setup.py create mode 100644 lib/py/src/TSCons.py create mode 100644 lib/py/src/Thrift.py create mode 100644 lib/py/src/__init__.py create mode 100644 lib/py/src/protocol/TBinaryProtocol.py create mode 100644 lib/py/src/protocol/TProtocol.py create mode 100644 lib/py/src/protocol/__init__.py create mode 100644 lib/py/src/protocol/fastbinary.c create mode 100644 lib/py/src/server/THttpServer.py create mode 100644 lib/py/src/server/TNonblockingServer.py create mode 100644 lib/py/src/server/TServer.py create mode 100644 lib/py/src/server/__init__.py create mode 100644 lib/py/src/transport/THttpClient.py create mode 100644 lib/py/src/transport/TSocket.py create mode 100644 lib/py/src/transport/TTransport.py create mode 100644 lib/py/src/transport/TTwisted.py create mode 100644 lib/py/src/transport/__init__.py create mode 100644 lib/rb/CHANGELOG create mode 100644 lib/rb/Makefile.am create mode 100644 lib/rb/Manifest create mode 100644 lib/rb/README create mode 100644 lib/rb/Rakefile create mode 100644 lib/rb/benchmark/Benchmark.thrift create mode 100644 lib/rb/benchmark/benchmark.rb create mode 100644 lib/rb/benchmark/client.rb create mode 100644 lib/rb/benchmark/server.rb create mode 100644 lib/rb/benchmark/thin_server.rb create mode 100644 lib/rb/ext/binary_protocol_accelerated.c create mode 100644 lib/rb/ext/binary_protocol_accelerated.h create mode 100644 lib/rb/ext/compact_protocol.c create mode 100644 lib/rb/ext/compact_protocol.h create mode 100644 lib/rb/ext/constants.h create mode 100644 lib/rb/ext/extconf.rb create mode 100644 lib/rb/ext/macros.h create mode 100644 lib/rb/ext/memory_buffer.c create mode 100644 lib/rb/ext/memory_buffer.h create mode 100644 lib/rb/ext/protocol.c create mode 100644 lib/rb/ext/protocol.h create mode 100644 lib/rb/ext/struct.c create mode 100644 lib/rb/ext/struct.h create mode 100644 lib/rb/ext/thrift_native.c create mode 100644 lib/rb/lib/thrift.rb create mode 100644 lib/rb/lib/thrift/client.rb create mode 100644 lib/rb/lib/thrift/core_ext.rb create mode 100644 lib/rb/lib/thrift/core_ext/fixnum.rb create mode 100644 lib/rb/lib/thrift/exceptions.rb create mode 100644 lib/rb/lib/thrift/processor.rb create mode 100644 lib/rb/lib/thrift/protocol/base_protocol.rb create mode 100644 lib/rb/lib/thrift/protocol/binary_protocol.rb create mode 100644 lib/rb/lib/thrift/protocol/binary_protocol_accelerated.rb create mode 100644 lib/rb/lib/thrift/protocol/compact_protocol.rb create mode 100644 lib/rb/lib/thrift/serializer/deserializer.rb create mode 100644 lib/rb/lib/thrift/serializer/serializer.rb create mode 100644 lib/rb/lib/thrift/server/base_server.rb create mode 100644 lib/rb/lib/thrift/server/mongrel_http_server.rb create mode 100644 lib/rb/lib/thrift/server/nonblocking_server.rb create mode 100644 lib/rb/lib/thrift/server/simple_server.rb create mode 100644 lib/rb/lib/thrift/server/thread_pool_server.rb create mode 100644 lib/rb/lib/thrift/server/threaded_server.rb create mode 100644 lib/rb/lib/thrift/struct.rb create mode 100644 lib/rb/lib/thrift/thrift_native.rb create mode 100644 lib/rb/lib/thrift/transport/base_server_transport.rb create mode 100644 lib/rb/lib/thrift/transport/base_transport.rb create mode 100644 lib/rb/lib/thrift/transport/buffered_transport.rb create mode 100644 lib/rb/lib/thrift/transport/framed_transport.rb create mode 100644 lib/rb/lib/thrift/transport/http_client_transport.rb create mode 100644 lib/rb/lib/thrift/transport/io_stream_transport.rb create mode 100644 lib/rb/lib/thrift/transport/memory_buffer_transport.rb create mode 100644 lib/rb/lib/thrift/transport/server_socket.rb create mode 100644 lib/rb/lib/thrift/transport/socket.rb create mode 100644 lib/rb/lib/thrift/transport/unix_server_socket.rb create mode 100644 lib/rb/lib/thrift/transport/unix_socket.rb create mode 100644 lib/rb/lib/thrift/types.rb create mode 100644 lib/rb/script/proto_benchmark.rb create mode 100644 lib/rb/script/read_struct.rb create mode 100644 lib/rb/script/write_struct.rb create mode 100644 lib/rb/setup.rb create mode 100644 lib/rb/spec/ThriftSpec.thrift create mode 100644 lib/rb/spec/base_protocol_spec.rb create mode 100644 lib/rb/spec/base_transport_spec.rb create mode 100644 lib/rb/spec/binary_protocol_accelerated_spec.rb create mode 100644 lib/rb/spec/binary_protocol_spec.rb create mode 100644 lib/rb/spec/binary_protocol_spec_shared.rb create mode 100644 lib/rb/spec/client_spec.rb create mode 100644 lib/rb/spec/compact_protocol_spec.rb create mode 100644 lib/rb/spec/exception_spec.rb create mode 100644 lib/rb/spec/http_client_spec.rb create mode 100644 lib/rb/spec/mongrel_http_server_spec.rb create mode 100644 lib/rb/spec/nonblocking_server_spec.rb create mode 100644 lib/rb/spec/processor_spec.rb create mode 100644 lib/rb/spec/serializer_spec.rb create mode 100644 lib/rb/spec/server_socket_spec.rb create mode 100644 lib/rb/spec/server_spec.rb create mode 100644 lib/rb/spec/socket_spec.rb create mode 100644 lib/rb/spec/socket_spec_shared.rb create mode 100644 lib/rb/spec/spec_helper.rb create mode 100644 lib/rb/spec/struct_spec.rb create mode 100644 lib/rb/spec/types_spec.rb create mode 100644 lib/rb/spec/unix_socket_spec.rb create mode 100644 lib/st/README create mode 100644 lib/st/thrift.st create mode 100755 print_version.sh create mode 100644 test/AllProtocolTests.cpp create mode 100644 test/AllProtocolTests.tcc create mode 100644 test/AnnotationTest.thrift create mode 100644 test/Benchmark.cpp create mode 100644 test/BrokenConstants.thrift create mode 100644 test/ConstantsDemo.thrift create mode 100644 test/DebugProtoTest.cpp create mode 100644 test/DebugProtoTest.thrift create mode 100644 test/DebugProtoTest_extras.cpp create mode 100644 test/DenseLinkingTest.thrift create mode 100644 test/DenseProtoTest.cpp create mode 100644 test/DocTest.thrift create mode 100755 test/FastbinaryTest.py create mode 100644 test/GenericHelpers.h create mode 100644 test/JSONProtoTest.cpp create mode 100644 test/JavaBeansTest.thrift create mode 100644 test/Makefile.am create mode 100644 test/ManyTypedefs.thrift create mode 100644 test/OptionalRequiredTest.cpp create mode 100644 test/OptionalRequiredTest.thrift create mode 100644 test/SmallTest.thrift create mode 100644 test/StressTest.thrift create mode 100644 test/TBufferBaseTest.cpp create mode 100644 test/TFDTransportTest.cpp create mode 100644 test/TMemoryBufferTest.cpp create mode 100644 test/TPipedTransportTest.cpp create mode 100644 test/ThriftTest.thrift create mode 100644 test/ThriftTest_extras.cpp create mode 100644 test/UnitTestMain.cpp create mode 100644 test/ZlibTest.cpp create mode 100644 test/cpp/Stress-test.mk create mode 100644 test/cpp/Thrift-test.mk create mode 100644 test/cpp/realloc/Makefile create mode 100644 test/cpp/realloc/realloc_test.c create mode 100644 test/cpp/src/TestClient.cpp create mode 100644 test/cpp/src/TestServer.cpp create mode 100644 test/cpp/src/main.cpp create mode 100644 test/cpp/src/nb-main.cpp create mode 100644 test/csharp/CSharpClient.cs create mode 100644 test/csharp/CSharpServer.cs create mode 100644 test/csharp/ThriftTest/Program.cs create mode 100644 test/csharp/ThriftTest/Properties/AssemblyInfo.cs create mode 100644 test/csharp/ThriftTest/TestClient.cs create mode 100644 test/csharp/ThriftTest/TestServer.cs create mode 100644 test/csharp/ThriftTest/ThriftTest.csproj create mode 100755 test/csharp/ThriftTest/maketest.sh create mode 100644 test/erl/Makefile create mode 100644 test/erl/src/stress_server.erl create mode 100644 test/erl/src/test_disklog.erl create mode 100644 test/erl/src/test_membuffer.erl create mode 100644 test/erl/src/test_server.erl create mode 100644 test/hs/Client.hs create mode 100644 test/hs/Server.hs create mode 100644 test/hs/runclient.sh create mode 100644 test/hs/runserver.sh create mode 100644 test/ocaml/Makefile create mode 100644 test/ocaml/client/Makefile create mode 100644 test/ocaml/client/TestClient.ml create mode 100644 test/ocaml/server/Makefile create mode 100644 test/ocaml/server/TestServer.ml create mode 100644 test/perl/Makefile create mode 100644 test/perl/TestClient.pl create mode 100644 test/php/Makefile create mode 100644 test/php/TestClient.php create mode 100644 test/php/TestInline.php create mode 100644 test/py/Makefile.am create mode 100755 test/py/RunClientServer.py create mode 100755 test/py/SerializationTest.py create mode 100755 test/py/TestClient.py create mode 100755 test/py/TestEof.py create mode 100755 test/py/TestServer.py create mode 100755 test/py/TestSocket.py create mode 100755 test/py/TestSyntax.py create mode 100755 test/py/explicit_module/runtest.sh create mode 100644 test/py/explicit_module/test1.thrift create mode 100644 test/py/explicit_module/test2.thrift create mode 100644 test/rb/Makefile.am create mode 100644 test/rb/benchmarks/protocol_benchmark.rb create mode 100644 test/rb/core/test_backwards_compatability.rb create mode 100644 test/rb/core/test_exceptions.rb create mode 100644 test/rb/core/transport/test_transport.rb create mode 100644 test/rb/fixtures/structs.rb create mode 100644 test/rb/generation/test_enum.rb create mode 100644 test/rb/generation/test_struct.rb create mode 100644 test/rb/integration/accelerated_buffered_client.rb create mode 100644 test/rb/integration/accelerated_buffered_server.rb create mode 100644 test/rb/integration/buffered_client.rb create mode 100644 test/rb/integration/simple_client.rb create mode 100644 test/rb/integration/simple_server.rb create mode 100644 test/rb/integration/test_simple_handler.rb create mode 100644 test/rb/test_helper.rb create mode 100644 test/rb/test_suite.rb create mode 100644 test/threads/Makefile create mode 100644 test/threads/ThreadsClient.cpp create mode 100644 test/threads/ThreadsServer.cpp create mode 100644 test/threads/ThreadsTest.thrift create mode 100644 tutorial/README create mode 100644 tutorial/cpp/CppClient.cpp create mode 100644 tutorial/cpp/CppServer.cpp create mode 100644 tutorial/cpp/Makefile create mode 100644 tutorial/erl/client.erl create mode 120000 tutorial/erl/client.sh create mode 100644 tutorial/erl/server.erl create mode 100755 tutorial/erl/server.sh create mode 100755 tutorial/java/JavaClient create mode 100755 tutorial/java/JavaServer create mode 100644 tutorial/java/build.xml create mode 100644 tutorial/java/src/JavaClient.java create mode 100644 tutorial/java/src/JavaServer.java create mode 100644 tutorial/perl/PerlClient.pl create mode 100755 tutorial/php/PhpClient.php create mode 100755 tutorial/php/PhpServer.php create mode 100755 tutorial/php/runserver.py create mode 100755 tutorial/py/PythonClient.py create mode 100755 tutorial/py/PythonServer.py create mode 100755 tutorial/rb/RubyClient.rb create mode 100755 tutorial/rb/RubyServer.rb create mode 100644 tutorial/shared.thrift create mode 100644 tutorial/tutorial.thrift diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..793d695b --- /dev/null +++ b/.gitignore @@ -0,0 +1,88 @@ +/Makefile +/Makefile.in +/aclocal.m4 +/autom4te.cache +/autoscan.log +/compiler/cpp/.deps +/compiler/cpp/Makefile +/compiler/cpp/Makefile.in +/compiler/cpp/thrift +/compiler/cpp/*.o +/compiler/cpp/thriftl.cc +/compiler/cpp/thrifty.cc +/compiler/cpp/thrifty.h +/compiler/cpp/version.h +/config.* +/configure +/configure.lineno +/configure.scan +/depcomp +/if/Makefile +/if/Makefile.in +/install-sh +/lib/Makefile +/lib/Makefile.in +/lib/erl/ebin +/lib/cpp/.deps +/lib/cpp/.libs +/lib/cpp/Makefile +/lib/cpp/Makefile.in +/lib/cpp/concurrency_test +/lib/cpp/*.o +/lib/cpp/*.la +/lib/cpp/*.lo +/lib/cpp/*.pc +/lib/csharp/Makefile +/lib/csharp/Makefile.in +/lib/java/Makefile +/lib/java/Makefile.in +/lib/java/build +/lib/java/gen-java +/lib/java/gen-javabean +/lib/java/libthrift.jar +/lib/perl/MANIFEST +/lib/perl/Makefile +/lib/perl/Makefile.in +/lib/perl/Makefile-perl.mk +/lib/perl/blib +/lib/perl/pm_to_blib +/lib/perl/test/Makefile +/lib/perl/test/Makefile.in +/lib/perl/test/gen-perl +/lib/py/Makefile +/lib/py/Makefile.in +/lib/py/build +/lib/rb/Makefile +/lib/rb/Makefile.in +/libtool +/ltmain.sh +/missing +/stamp-h1 +/test/.deps +/test/.libs +/test/*.o +/test/*.la +/test/*.lo +/test/Benchmark +/test/DebugProtoTest +/test/JSONProtoTest +/test/TFDTransportTest +/test/TPipedTransportTest +/test/UnitTests +/test/Makefile +/test/Makefile.in +/test/OptionalRequiredTest +/test/ReflectionTest +/test/gen-cpp +/test/java/Makefile +/test/java/Makefile.am +/test/java/Makefile.in +/test/java/build +/test/java/gen-java +/test/java/thrifttest.jar +/test/py/Makefile +/test/py/Makefile.in +/test/py/gen-py +/test/rb/Makefile +/test/rb/Makefile.in +/ylwrap diff --git a/CHANGES b/CHANGES new file mode 100644 index 00000000..e4d81933 --- /dev/null +++ b/CHANGES @@ -0,0 +1,35 @@ +Thrift Changelog + +Version 0.1.0 RC1 / Unreleased + +Compatibility Breaking Changes: + C++: + * It's quite possible that regenerating code and rebuilding will be + required. Make sure your headers match your libs! + + Java: + + Python: + + Ruby: + * Generated files now have underscored names [THRIFT-421] + * The library has been rearranged to be more Ruby-like [THRIFT-276] + + Erlang: + * Generated code will have to be regenerated, and the new code will + have to be deployed atomically with the new library code [THRIFT-136] + + +New Features and Bug Fixes: + C++: + * Support for TCompactProtocol [THRIFT-333] + + Java: + * Support for TCompactProtocol [THRIFT-110] + + Python: + * Support for Twisted [THRIFT-148] + + Ruby: + * Support for TCompactProtocol [THRIFT-332] + diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 00000000..fd954f8d --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,77 @@ +Chad Walters +-TJSONProtocol for C++ and Java + +Nitay +-Support for "make check" + +William Morgan +-Miscellaneous Ruby improvements + +Ben Maurer +-Restructuring the way Autoconf is used + +Patrick Collison +-Smalltalk bindings + +Dave Simpson +-Better support for connection tracking in the C++ server +-Miscellaneous fixes + +Igor Afanasyev +-Perl HttpClient and bugfixes + +Todd Berman +-MinGW port of the compiler +-C# bindings +-MS build task + +---------------- +Release 20070917 +---------------- + +Dave Engberg +-JavaBean/JavaDoc enhancements + +Andrew McGeachie +-Cocoa/Objective-C support + +Ben Maurer +-Python performance enhancements, fastbinary support + +Andrew Lutomirski +-Added optional/required keywords for C++ objects +-Added comparison operators for C++ structs + +Johan Oskarsson +-Java findbugs compliance fixes + +Paul Saab +-IPv6 support for TSocket in C++/Python + +Kevin Clark +-Significant overhaul of Ruby code generation and servers + +Simon Forman +-TProcessorFactory abstraction for Java servers + +Jake Luciani +-Perl code generation, libraries, test code + +David Reiss +-strings.h include fix for bzero +-endianness fixes on TBinaryProtocol double serialization +-improved ntohll,htonll implementation + +Dan Li +-Java TestServer and Tutorial Fixes + +Kevin Ko +-Fix for unnecessary std::string copy construction in Protocol/Exception + +Paul Querna +-Autoconf error message fix for libevent detection +-clock_gettime implementation for OSX + +---------------- +Release 20070401 +---------------- diff --git a/DISCLAIMER b/DISCLAIMER new file mode 100644 index 00000000..de7e7ea2 --- /dev/null +++ b/DISCLAIMER @@ -0,0 +1,6 @@ +Apache Thrift is an effort undergoing incubation at The Apache Software Foundation (ASF), +sponsored by the Incubator PMC. Incubation is required of all newly accepted projects +until a further review indicates that the infrastructure, communications, and decision +making process have stabilized in a manner consistent with other successful ASF projects. +While incubation status is not necessarily a reflection of the completeness or stability +of the code, it does indicate that the project has yet to be fully endorsed by the ASF. diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 00000000..bd566010 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +ACLOCAL_AMFLAGS = -I ./aclocal + +SUBDIRS = compiler/cpp lib test + +dist-hook: + find $(distdir) -name '.[a-zA-Z0-9]*' | xargs rm -rf + +EXTRA_DIST = bootstrap.sh cleanup.sh print_version.sh doc tutorial contrib \ + CONTRIBUTORS LICENSE CHANGES DISCLAIMER NOTICE diff --git a/NEWS b/NEWS new file mode 100644 index 00000000..2369f61d --- /dev/null +++ b/NEWS @@ -0,0 +1,79 @@ +Release Notes for Thrift 20080411 + +.equals and .hashCode() for Java scturcts (developed by dreiss). + +Improvments to the C++ TSocketPool (developed by akhil). + +PHP (de)serialization extension (developed by dweatherford). + +Add fb303 to contrib (developed by Facebook). + +TJSONProtocol for C++ and Java (contributed by Chad Walters of Powerset). + +Support for "make check" and better tests (contributed by Nitay). + +Smalltalk support (contributed by Patrick Collison). + +Dave Simpson +Better support for connection tracking in the C++ server (contributed by +Dave Simpson of Powerset). + +Perl HttpClient (contributed by Igor Afanasyev of Evernote). + +C# support (contributed by Todd Berman of imeem). + +MinGW port of the compiler (contributed by Todd Berman of imeem). + +Tons of small improvements and bug fixes. + + +Release Notes for Thrift 20070917 + +TBinaryProtocol now includes a protocol version number in messaged. +This is a non-backwards-compatible change. Please see the +TBinaryProtocol constructor for strictRead_ and strictWrite_. + +Add binary type to support non-text "strings" in Java. + +TSocketPool for C++ (developed by jsobel). + +Syntax highlighting for vim and emacs (developed by hzhao and martin). + +Perl support (contributed by Jake Luciani). + +Erlang support (developed by cpiro). + +Ruby API overhaul (contributed by Kevin Clark of Powerset). + +IPv6 support in C++ and Python (contributed by Paul Saab of Powerset). + +Read/Write locks (developed by boz). + +OCaml support (developed by iproctor). + +Human-readable strings from Thrift structures in C++ (developed by dreiss). + +Haskell support (developed by iproctor). + +Support for optional fields in C++ (contributed by Andy Lutomirsky). + +Support for operator== for Thrift structures (contributed by Andy Lutomirsky). + +Python/C module for fast (de)serialization (contributed by Ben Maurer). + +Limited reflection for C++ services (developed by dreiss). + +Python library installation defaults to /usr, override with PY_PREFIX. + +Support for Javabean-style Java classes (contributed by Dave Engberg). + +TDenseProtocol for C++ (experimental way to shrink structures) +(developed by dreiss). + +Cocoa/Objective C support (contributed by Andrew McGeachie). + +Thrift now builds without libevent. + +TZlibTransport for C++ (compress serialized structures) (developed by dreiss). + +Tons of small improvements and bug fixes. diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..b5186316 --- /dev/null +++ b/NOTICE @@ -0,0 +1,26 @@ +Apache Thrift +Copyright 2006-2009 The Apache Software Foundation, et al. + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Some files in this distribution are distributed under different terms +from the rest of Apache Thrift. Please see individual files for +license information. + +In addition, the following unlabelled files are distributed under +specific terms. Please see the "doc" directory for the text of their +licenses. + + lib/rb/setup.rb: GNU Lesser General Public License 2.1 (lgpl-2.1.txt) + lib/ocaml/OCamlMakefile: GNU Lesser General Public License 2.1 (lgpl-2.1.txt) + lib/ocaml/README-OCamlMakefile: GNU Lesser General Public License 2.1 (lgpl-2.1.txt) + lib/erl/build/beamver: MIT License (otp-base-license.txt) + lib/erl/build/buildtargets.mk: MIT License (otp-base-license.txt) + lib/erl/build/colors.mk: MIT License (otp-base-license.txt) + lib/erl/build/docs.mk: MIT License (otp-base-license.txt) + lib/erl/build/mime.types: MIT License (otp-base-license.txt) + lib/erl/build/otp.mk: MIT License (otp-base-license.txt) + lib/erl/build/otp_subdir.mk: MIT License (otp-base-license.txt) + lib/erl/build/raw_test.mk: MIT License (otp-base-license.txt) + lib/erl/src/Makefile: MIT License (otp-base-license.txt) diff --git a/README b/README new file mode 100644 index 00000000..adf8af47 --- /dev/null +++ b/README @@ -0,0 +1,137 @@ +Apache Thrift (an Apache Incubator project) + +Last Modified: 2009-Jan-30 + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Introduction +============ + +Thrift is a lightweight, language-independent software stack with an +associated code generation mechanism for RPC. Thrift provides clean +abstractions for data transport, data serialization, and application +level processing. The code generation system takes a simple definition +language as its input and generates code across programming languages that +uses the abstracted stack to build interoperable RPC clients and servers. + +Thrift is specifically designed to support non-atomic version changes +across client and server code. + +For more details on Thrift's design and implementation, take a gander at +the Thrift whitepaper included in this distribution or at the README files +in your particular subdirectory of interest. + +Heirarchy +========= + +thrift/ + + compiler/ + Contains the Thrift compiler, implemented in C++. + + lib/ + Contains the Thrift software library implementation, subdivided by + language of implementation. + + cpp/ + java/ + php/ + py/ + rb/ + + test/ + + Contains sample Thrift files and test code across the target programming + languages. + + tutorial/ + + Contains a basic tutorial that will teach you how to develop software + using Thrift. + +Requirements +============ + +See http://wiki.apache.org/thrift/ThriftRequirements for +an up-to-date list of build requirements. + +Resources +========= + +More information about Thrift can be obtained on the Thrift webpage at: + + http://incubator.apache.org/thrift + +Acknowledgments +=============== + +Thrift was inspired by pillar, a lightweight RPC tool written by Adam D'Angelo, +and also by Google's protocol buffers. + +Installation +============ + +If you are building from the first time out of the source repository, you will +need to generate the configure scripts. (This is not necessary if you +downloaded a tarball.) From the top directory, do: + + ./bootstrap.sh + +Once the configure scripts are generated, thrift can be configured. +From the top directory, do: + + ./configure + +You may need to specify the location of the boost files explicitly. +If you installed boost in /usr/local, you would run configure as follows: + + ./configure --with-boost=/usr/local + +Note that by default the thrift C++ library is typically built with debugging +symbols included. If you want to customize these options you should use the +CXXFLAGS option in configure, as such: + + ./configure CXXFLAGS='-g -O2' + ./configure CFLAGS='-g -O2' + ./configure CPPFLAGS='-DDEBUG_MY_FEATURE' + +Run ./configure --help to see other configuration options + +Please be aware that the Python library will ignore the --prefix option +and just install wherever Python's distutils puts it (usually along +the lines of /usr/lib/pythonX.Y/site-packages/). If you need to control +where the Python modules are installed, set the PY_PREFIX variable. +(DESTDIR is respected for Python and C++.) + +Make thrift: + + make + +From the top directory, become superuser and do: + + make install + +Note that some language packages must be installed manually using build tools +better suited to those languages (at the time of this writing, this applies +to Java, Ruby, PHP). + +Look for the README file in the lib// folder for more details on the +installation of each language library package. diff --git a/aclocal/ax_boost_base.m4 b/aclocal/ax_boost_base.m4 new file mode 100644 index 00000000..e56bb738 --- /dev/null +++ b/aclocal/ax_boost_base.m4 @@ -0,0 +1,198 @@ +##### http://autoconf-archive.cryp.to/ax_boost_base.html +# +# SYNOPSIS +# +# AX_BOOST_BASE([MINIMUM-VERSION]) +# +# DESCRIPTION +# +# Test for the Boost C++ libraries of a particular version (or newer) +# +# If no path to the installed boost library is given the macro +# searchs under /usr, /usr/local, /opt and /opt/local and evaluates +# the $BOOST_ROOT environment variable. Further documentation is +# available at . +# +# This macro calls: +# +# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS) +# +# And sets: +# +# HAVE_BOOST +# +# LAST MODIFICATION +# +# 2007-07-28 +# +# COPYLEFT +# +# Copyright (c) 2007 Thomas Porschberg +# +# Copying and distribution of this file, with or without +# modification, are permitted in any medium without royalty provided +# the copyright notice and this notice are preserved. + +AC_DEFUN([AX_BOOST_BASE], +[ +AC_ARG_WITH([boost], + AS_HELP_STRING([--with-boost@<:@=DIR@:>@], [use boost (default is yes) - it is possible to specify the root directory for boost (optional)]), + [ + if test "$withval" = "no"; then + want_boost="no" + elif test "$withval" = "yes"; then + want_boost="yes" + ac_boost_path="" + else + want_boost="yes" + ac_boost_path="$withval" + fi + ], + [want_boost="yes"]) + +if test "x$want_boost" = "xyes"; then + boost_lib_version_req=ifelse([$1], ,1.20.0,$1) + boost_lib_version_req_shorten=`expr $boost_lib_version_req : '\([[0-9]]*\.[[0-9]]*\)'` + boost_lib_version_req_major=`expr $boost_lib_version_req : '\([[0-9]]*\)'` + boost_lib_version_req_minor=`expr $boost_lib_version_req : '[[0-9]]*\.\([[0-9]]*\)'` + boost_lib_version_req_sub_minor=`expr $boost_lib_version_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'` + if test "x$boost_lib_version_req_sub_minor" = "x" ; then + boost_lib_version_req_sub_minor="0" + fi + WANT_BOOST_VERSION=`expr $boost_lib_version_req_major \* 100000 \+ $boost_lib_version_req_minor \* 100 \+ $boost_lib_version_req_sub_minor` + AC_MSG_CHECKING(for boostlib >= $boost_lib_version_req) + succeeded=no + + dnl first we check the system location for boost libraries + dnl this location ist chosen if boost libraries are installed with the --layout=system option + dnl or if you install boost with RPM + if test "$ac_boost_path" != ""; then + BOOST_LDFLAGS="-L$ac_boost_path/lib" + BOOST_CPPFLAGS="-I$ac_boost_path/include" + else + for ac_boost_path_tmp in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path_tmp/include/boost" && test -r "$ac_boost_path_tmp/include/boost"; then + BOOST_LDFLAGS="-L$ac_boost_path_tmp/lib" + BOOST_CPPFLAGS="-I$ac_boost_path_tmp/include" + break; + fi + done + fi + + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_LANG_PUSH(C++) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + succeeded=yes + found_system=yes + ],[ + ]) + AC_LANG_POP([C++]) + + + + dnl if we found no boost with system layout we search for boost libraries + dnl built and installed without the --layout=system option or for a staged(not installed) version + if test "x$succeeded" != "xyes"; then + _version=0 + if test "$ac_boost_path" != ""; then + BOOST_LDFLAGS="-L$ac_boost_path/lib" + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + fi + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$ac_boost_path/include/boost-$VERSION_UNDERSCORE" + done + fi + else + for ac_boost_path in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + best_path=$ac_boost_path + fi + done + fi + done + + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE" + BOOST_LDFLAGS="-L$best_path/lib" + + if test "x$BOOST_ROOT" != "x"; then + if test -d "$BOOST_ROOT" && test -r "$BOOST_ROOT" && test -d "$BOOST_ROOT/stage/lib" && test -r "$BOOST_ROOT/stage/lib"; then + version_dir=`expr //$BOOST_ROOT : '.*/\(.*\)'` + stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'` + stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'` + V_CHECK=`expr $stage_version_shorten \>\= $_version` + if test "$V_CHECK" = "1" ; then + AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT) + BOOST_CPPFLAGS="-I$BOOST_ROOT" + BOOST_LDFLAGS="-L$BOOST_ROOT/stage/lib" + fi + fi + fi + fi + + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_LANG_PUSH(C++) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + succeeded=yes + found_system=yes + ],[ + ]) + AC_LANG_POP([C++]) + fi + + if test "$succeeded" != "yes" ; then + if test "$_version" = "0" ; then + AC_MSG_ERROR([[We could not detect the boost libraries (version $boost_lib_version_req_shorten or higher). If you have a staged boost library (still not installed) please specify \$BOOST_ROOT in your environment and do not give a PATH to --with-boost option. If you are sure you have boost installed, then check your version number looking in . See http://randspringer.de/boost for more documentation.]]) + else + AC_MSG_NOTICE([Your boost libraries seems to old (version $_version).]) + fi + else + AC_SUBST(BOOST_CPPFLAGS) + AC_SUBST(BOOST_LDFLAGS) + AC_DEFINE(HAVE_BOOST,,[define if the Boost library is available]) + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" +fi + +]) diff --git a/aclocal/ax_javac_and_java.m4 b/aclocal/ax_javac_and_java.m4 new file mode 100644 index 00000000..3c8577f4 --- /dev/null +++ b/aclocal/ax_javac_and_java.m4 @@ -0,0 +1,84 @@ +dnl @synopsis AX_JAVAC_AND_JAVA +dnl +dnl Test for the presence of a JDK. +dnl +dnl If "JAVA" is defined in the environment, that will be the only +dnl java command tested. Otherwise, a hard-coded list will be used. +dnl Similarly for "JAVAC". +dnl +dnl This macro does not currenly support testing for a particular +dnl Java version, the presence of a particular class, testing for +dnl only one of "java" and "javac", or compiling or running +dnl user-provided Java code. +dnl +dnl After AX_JAVAC_AND_JAVA runs, the shell variables "success" and +dnl "ax_javac_and_java" are set to "yes" or "no", and "JAVAC" and +dnl "JAVA" are set to the appropriate commands. +dnl +dnl @category Java +dnl @version 2009-02-09 +dnl @license AllPermissive +dnl +dnl Copyright (C) 2009 David Reiss +dnl Copying and distribution of this file, with or without modification, +dnl are permitted in any medium without royalty provided the copyright +dnl notice and this notice are preserved. + + +AC_DEFUN([AX_JAVAC_AND_JAVA], + [ + + dnl Hard-coded default commands to test. + JAVAC_PROGS="javac,jikes,gcj -C" + JAVA_PROGS="java,kaffe" + + dnl Allow the user to specify an alternative. + if test -n "$JAVAC" ; then + JAVAC_PROGS="$JAVAC" + fi + if test -n "$JAVA" ; then + JAVA_PROGS="$JAVA" + fi + + AC_MSG_CHECKING(for javac and java) + + echo "public class configtest_ax_javac_and_java { public static void main(String args@<:@@:>@) { } }" > configtest_ax_javac_and_java.java + success=no + oIFS="$IFS" + + IFS="," + for JAVAC in $JAVAC_PROGS ; do + IFS="$oIFS" + + echo "Running \"$JAVAC configtest_ax_javac_and_java.java\"" >&AS_MESSAGE_LOG_FD + if $JAVAC configtest_ax_javac_and_java.java >&AS_MESSAGE_LOG_FD 2>&1 ; then + + IFS="," + for JAVA in $JAVA_PROGS ; do + IFS="$oIFS" + + echo "Running \"$JAVA configtest_ax_javac_and_java\"" >&AS_MESSAGE_LOG_FD + if $JAVA configtest_ax_javac_and_java >&AS_MESSAGE_LOG_FD 2>&1 ; then + success=yes + break 2 + fi + + done + + fi + + done + + rm -f configtest_ax_javac_and_java.java configtest_ax_javac_and_java.class + + if test "$success" != "yes" ; then + AC_MSG_RESULT(no) + JAVAC="" + JAVA="" + else + AC_MSG_RESULT(yes) + fi + + ax_javac_and_java="$success" + + ]) diff --git a/aclocal/ax_lib_event.m4 b/aclocal/ax_lib_event.m4 new file mode 100644 index 00000000..3a48156a --- /dev/null +++ b/aclocal/ax_lib_event.m4 @@ -0,0 +1,173 @@ +dnl @synopsis AX_LIB_EVENT([MINIMUM-VERSION]) +dnl +dnl Test for the libevent library of a particular version (or newer). +dnl +dnl If no path to the installed libevent is given, the macro will first try +dnl using no -I or -L flags, then searches under /usr, /usr/local, /opt, +dnl and /opt/libevent. +dnl If these all fail, it will try the $LIBEVENT_ROOT environment variable. +dnl +dnl This macro requires that #include works and defines u_char. +dnl +dnl This macro calls: +dnl AC_SUBST(LIBEVENT_CPPFLAGS) +dnl AC_SUBST(LIBEVENT_LDFLAGS) +dnl AC_SUBST(LIBEVENT_LIBS) +dnl +dnl And (if libevent is found): +dnl AC_DEFINE(HAVE_LIBEVENT) +dnl +dnl It also leaves the shell variables "success" and "ax_have_libevent" +dnl set to "yes" or "no". +dnl +dnl NOTE: This macro does not currently work for cross-compiling, +dnl but it can be easily modified to allow it. (grep "cross"). +dnl +dnl @category InstalledPackages +dnl @category C +dnl @version 2007-09-12 +dnl @license AllPermissive +dnl +dnl Copyright (C) 2009 David Reiss +dnl Copying and distribution of this file, with or without modification, +dnl are permitted in any medium without royalty provided the copyright +dnl notice and this notice are preserved. + +dnl Input: ax_libevent_path, WANT_LIBEVENT_VERSION +dnl Output: success=yes/no +AC_DEFUN([AX_LIB_EVENT_DO_CHECK], + [ + # Save our flags. + CPPFLAGS_SAVED="$CPPFLAGS" + LDFLAGS_SAVED="$LDFLAGS" + LIBS_SAVED="$LIBS" + LD_LIBRARY_PATH_SAVED="$LD_LIBRARY_PATH" + + # Set our flags if we are checking a specific directory. + if test -n "$ax_libevent_path" ; then + LIBEVENT_CPPFLAGS="-I$ax_libevent_path/include" + LIBEVENT_LDFLAGS="-L$ax_libevent_path/lib" + LD_LIBRARY_PATH="$ax_libevent_path/lib:$LD_LIBRARY_PATH" + else + LIBEVENT_CPPFLAGS="" + LIBEVENT_LDFLAGS="" + fi + + # Required flag for libevent. + LIBEVENT_LIBS="-levent" + + # Prepare the environment for compilation. + CPPFLAGS="$CPPFLAGS $LIBEVENT_CPPFLAGS" + LDFLAGS="$LDFLAGS $LIBEVENT_LDFLAGS" + LIBS="$LIBS $LIBEVENT_LIBS" + export CPPFLAGS + export LDFLAGS + export LIBS + export LD_LIBRARY_PATH + + success=no + + # Compile, link, and run the program. This checks: + # - event.h is available for including. + # - event_get_version() is available for linking. + # - The event version string is lexicographically greater + # than the required version. + AC_LANG_PUSH([C]) + dnl This can be changed to AC_LINK_IFELSE if you are cross-compiling, + dnl but then the version cannot be checked. + AC_RUN_IFELSE([AC_LANG_PROGRAM([[ + #include + #include + ]], [[ + const char* lib_version = event_get_version(); + const char* wnt_version = "$WANT_LIBEVENT_VERSION"; + for (;;) { + /* If we reached the end of the want version. We have it. */ + if (*wnt_version == '\0') { + return 0; + } + /* If the want version continues but the lib version does not, */ + /* we are missing a letter. We don't have it. */ + if (*lib_version == '\0') { + return 1; + } + /* If we have greater than what we want. We have it. */ + if (*lib_version > *wnt_version) { + return 0; + } + /* If we have less, we don't. */ + if (*lib_version < *wnt_version) { + return 1; + } + lib_version++; + wnt_version++; + } + return 0; + ]])], [ + success=yes + ]) + AC_LANG_POP([C]) + + # Restore flags. + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" + LIBS="$LIBS_SAVED" + LD_LIBRARY_PATH="$LD_LIBRARY_PATH_SAVED" + ]) + + +AC_DEFUN([AX_LIB_EVENT], + [ + + dnl Allow search path to be overridden on the command line. + AC_ARG_WITH([libevent], + AS_HELP_STRING([--with-libevent@<:@=DIR@:>@], [use libevent (default is yes) - it is possible to specify an alternate root directory for libevent]), + [ + if test "x$withval" = "xno"; then + want_libevent="no" + elif test "x$withval" = "xyes"; then + want_libevent="yes" + ax_libevent_path="" + else + want_libevent="yes" + ax_libevent_path="$withval" + fi + ], + [ want_libevent="yes" ; ax_libevent_path="" ]) + + + if test "$want_libevent" = "yes"; then + WANT_LIBEVENT_VERSION=ifelse([$1], ,1.2,$1) + + AC_MSG_CHECKING(for libevent >= $WANT_LIBEVENT_VERSION) + + # Run tests. + if test -n "$ax_libevent_path"; then + AX_LIB_EVENT_DO_CHECK + else + for ax_libevent_path in "" /usr /usr/local /opt /opt/local /opt/libevent "$LIBEVENT_ROOT" ; do + AX_LIB_EVENT_DO_CHECK + if test "$success" = "yes"; then + break; + fi + done + fi + + if test "$success" != "yes" ; then + AC_MSG_RESULT(no) + LIBEVENT_CPPFLAGS="" + LIBEVENT_LDFLAGS="" + LIBEVENT_LIBS="" + else + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_LIBEVENT,,[define if libevent is available]) + fi + + ax_have_libevent="$success" + + AC_SUBST(LIBEVENT_CPPFLAGS) + AC_SUBST(LIBEVENT_LDFLAGS) + AC_SUBST(LIBEVENT_LIBS) + fi + + ]) diff --git a/aclocal/ax_lib_zlib.m4 b/aclocal/ax_lib_zlib.m4 new file mode 100644 index 00000000..8c10ab41 --- /dev/null +++ b/aclocal/ax_lib_zlib.m4 @@ -0,0 +1,173 @@ +dnl @synopsis AX_LIB_ZLIB([MINIMUM-VERSION]) +dnl +dnl Test for the libz library of a particular version (or newer). +dnl +dnl If no path to the installed zlib is given, the macro will first try +dnl using no -I or -L flags, then searches under /usr, /usr/local, /opt, +dnl and /opt/zlib. +dnl If these all fail, it will try the $ZLIB_ROOT environment variable. +dnl +dnl This macro calls: +dnl AC_SUBST(ZLIB_CPPFLAGS) +dnl AC_SUBST(ZLIB_LDFLAGS) +dnl AC_SUBST(ZLIB_LIBS) +dnl +dnl And (if zlib is found): +dnl AC_DEFINE(HAVE_ZLIB) +dnl +dnl It also leaves the shell variables "success" and "ax_have_zlib" +dnl set to "yes" or "no". +dnl +dnl NOTE: This macro does not currently work for cross-compiling, +dnl but it can be easily modified to allow it. (grep "cross"). +dnl +dnl @category InstalledPackages +dnl @category C +dnl @version 2007-09-12 +dnl @license AllPermissive +dnl +dnl Copyright (C) 2009 David Reiss +dnl Copying and distribution of this file, with or without modification, +dnl are permitted in any medium without royalty provided the copyright +dnl notice and this notice are preserved. + +dnl Input: ax_zlib_path, WANT_ZLIB_VERSION +dnl Output: success=yes/no +AC_DEFUN([AX_LIB_ZLIB_DO_CHECK], + [ + # Save our flags. + CPPFLAGS_SAVED="$CPPFLAGS" + LDFLAGS_SAVED="$LDFLAGS" + LIBS_SAVED="$LIBS" + LD_LIBRARY_PATH_SAVED="$LD_LIBRARY_PATH" + + # Set our flags if we are checking a specific directory. + if test -n "$ax_zlib_path" ; then + ZLIB_CPPFLAGS="-I$ax_zlib_path/include" + ZLIB_LDFLAGS="-L$ax_zlib_path/lib" + LD_LIBRARY_PATH="$ax_zlib_path/lib:$LD_LIBRARY_PATH" + else + ZLIB_CPPFLAGS="" + ZLIB_LDFLAGS="" + fi + + # Required flag for zlib. + ZLIB_LIBS="-lz" + + # Prepare the environment for compilation. + CPPFLAGS="$CPPFLAGS $ZLIB_CPPFLAGS" + LDFLAGS="$LDFLAGS $ZLIB_LDFLAGS" + LIBS="$LIBS $ZLIB_LIBS" + export CPPFLAGS + export LDFLAGS + export LIBS + export LD_LIBRARY_PATH + + success=no + + # Compile, link, and run the program. This checks: + # - zlib.h is available for including. + # - zlibVersion() is available for linking. + # - ZLIB_VERNUM is greater than or equal to the desired version. + # - ZLIB_VERSION (defined in zlib.h) matches zlibVersion() + # (defined in the library). + AC_LANG_PUSH([C]) + dnl This can be changed to AC_LINK_IFELSE if you are cross-compiling. + AC_RUN_IFELSE([AC_LANG_PROGRAM([[ + #include + #if ZLIB_VERNUM >= 0x$WANT_ZLIB_VERSION + #else + # error zlib is too old + #endif + ]], [[ + const char* lib_version = zlibVersion(); + const char* hdr_version = ZLIB_VERSION; + for (;;) { + if (*lib_version != *hdr_version) { + /* If this happens, your zlib header doesn't match your zlib */ + /* library. That is really bad. */ + return 1; + } + if (*lib_version == '\0') { + break; + } + lib_version++; + hdr_version++; + } + return 0; + ]])], [ + success=yes + ]) + AC_LANG_POP([C]) + + # Restore flags. + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" + LIBS="$LIBS_SAVED" + LD_LIBRARY_PATH="$LD_LIBRARY_PATH_SAVED" + ]) + + +AC_DEFUN([AX_LIB_ZLIB], + [ + + dnl Allow search path to be overridden on the command line. + AC_ARG_WITH([zlib], + AS_HELP_STRING([--with-zlib@<:@=DIR@:>@], [use zlib (default is yes) - it is possible to specify an alternate root directory for zlib]), + [ + if test "x$withval" = "xno"; then + want_zlib="no" + elif test "x$withval" = "xyes"; then + want_zlib="yes" + ax_zlib_path="" + else + want_zlib="yes" + ax_zlib_path="$withval" + fi + ], + [want_zlib="yes" ; ax_zlib_path="" ]) + + + if test "$want_zlib" = "yes"; then + # Parse out the version. + zlib_version_req=ifelse([$1], ,1.2.3,$1) + zlib_version_req_major=`expr $zlib_version_req : '\([[0-9]]*\)'` + zlib_version_req_minor=`expr $zlib_version_req : '[[0-9]]*\.\([[0-9]]*\)'` + zlib_version_req_patch=`expr $zlib_version_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'` + if test -z "$zlib_version_req_patch" ; then + zlib_version_req_patch="0" + fi + WANT_ZLIB_VERSION=`expr $zlib_version_req_major \* 1000 \+ $zlib_version_req_minor \* 100 \+ $zlib_version_req_patch \* 10` + + AC_MSG_CHECKING(for zlib >= $zlib_version_req) + + # Run tests. + if test -n "$ax_zlib_path"; then + AX_LIB_ZLIB_DO_CHECK + else + for ax_zlib_path in "" /usr /usr/local /opt /opt/zlib "$ZLIB_ROOT" ; do + AX_LIB_ZLIB_DO_CHECK + if test "$success" = "yes"; then + break; + fi + done + fi + + if test "$success" != "yes" ; then + AC_MSG_RESULT(no) + ZLIB_CPPFLAGS="" + ZLIB_LDFLAGS="" + ZLIB_LIBS="" + else + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_ZLIB,,[define if zlib is available]) + fi + + ax_have_zlib="$success" + + AC_SUBST(ZLIB_CPPFLAGS) + AC_SUBST(ZLIB_LDFLAGS) + AC_SUBST(ZLIB_LIBS) + fi + + ]) diff --git a/aclocal/ax_signed_right_shift.m4 b/aclocal/ax_signed_right_shift.m4 new file mode 100644 index 00000000..01952338 --- /dev/null +++ b/aclocal/ax_signed_right_shift.m4 @@ -0,0 +1,127 @@ +dnl @synopsis AX_SIGNED_RIGHT_SHIFT +dnl +dnl Tests the behavior of a right shift on a negative signed int. +dnl +dnl This macro calls: +dnl AC_DEFINE(SIGNED_RIGHT_SHIFT_IS) +dnl AC_DEFINE(ARITHMETIC_RIGHT_SHIFT) +dnl AC_DEFINE(LOGICAL_RIGHT_SHIFT) +dnl AC_DEFINE(UNKNOWN_RIGHT_SHIFT) +dnl +dnl SIGNED_RIGHT_SHIFT_IS will be equal to one of the other macros. +dnl It also leaves the shell variables "ax_signed_right_shift" +dnl set to "arithmetic", "logical", or "unknown". +dnl +dnl NOTE: This macro does not work for cross-compiling. +dnl +dnl @category C +dnl @version 2009-03-25 +dnl @license AllPermissive +dnl +dnl Copyright (C) 2009 David Reiss +dnl Copying and distribution of this file, with or without modification, +dnl are permitted in any medium without royalty provided the copyright +dnl notice and this notice are preserved. + +AC_DEFUN([AX_SIGNED_RIGHT_SHIFT], + [ + + AC_MSG_CHECKING(the behavior of a signed right shift) + + success_arithmetic=no + AC_RUN_IFELSE([AC_LANG_PROGRAM([[]], [[ + return + /* 0xffffffff */ + -1 >> 1 != -1 || + -1 >> 2 != -1 || + -1 >> 3 != -1 || + -1 >> 4 != -1 || + -1 >> 8 != -1 || + -1 >> 16 != -1 || + -1 >> 24 != -1 || + -1 >> 31 != -1 || + /* 0x80000000 */ + (-2147483647 - 1) >> 1 != -1073741824 || + (-2147483647 - 1) >> 2 != -536870912 || + (-2147483647 - 1) >> 3 != -268435456 || + (-2147483647 - 1) >> 4 != -134217728 || + (-2147483647 - 1) >> 8 != -8388608 || + (-2147483647 - 1) >> 16 != -32768 || + (-2147483647 - 1) >> 24 != -128 || + (-2147483647 - 1) >> 31 != -1 || + /* 0x90800000 */ + -1870659584 >> 1 != -935329792 || + -1870659584 >> 2 != -467664896 || + -1870659584 >> 3 != -233832448 || + -1870659584 >> 4 != -116916224 || + -1870659584 >> 8 != -7307264 || + -1870659584 >> 16 != -28544 || + -1870659584 >> 24 != -112 || + -1870659584 >> 31 != -1 || + 0; + ]])], [ + success_arithmetic=yes + ]) + + + success_logical=no + AC_RUN_IFELSE([AC_LANG_PROGRAM([[]], [[ + return + /* 0xffffffff */ + -1 >> 1 != (signed)((unsigned)-1 >> 1) || + -1 >> 2 != (signed)((unsigned)-1 >> 2) || + -1 >> 3 != (signed)((unsigned)-1 >> 3) || + -1 >> 4 != (signed)((unsigned)-1 >> 4) || + -1 >> 8 != (signed)((unsigned)-1 >> 8) || + -1 >> 16 != (signed)((unsigned)-1 >> 16) || + -1 >> 24 != (signed)((unsigned)-1 >> 24) || + -1 >> 31 != (signed)((unsigned)-1 >> 31) || + /* 0x80000000 */ + (-2147483647 - 1) >> 1 != (signed)((unsigned)(-2147483647 - 1) >> 1) || + (-2147483647 - 1) >> 2 != (signed)((unsigned)(-2147483647 - 1) >> 2) || + (-2147483647 - 1) >> 3 != (signed)((unsigned)(-2147483647 - 1) >> 3) || + (-2147483647 - 1) >> 4 != (signed)((unsigned)(-2147483647 - 1) >> 4) || + (-2147483647 - 1) >> 8 != (signed)((unsigned)(-2147483647 - 1) >> 8) || + (-2147483647 - 1) >> 16 != (signed)((unsigned)(-2147483647 - 1) >> 16) || + (-2147483647 - 1) >> 24 != (signed)((unsigned)(-2147483647 - 1) >> 24) || + (-2147483647 - 1) >> 31 != (signed)((unsigned)(-2147483647 - 1) >> 31) || + /* 0x90800000 */ + -1870659584 >> 1 != (signed)((unsigned)-1870659584 >> 1) || + -1870659584 >> 2 != (signed)((unsigned)-1870659584 >> 2) || + -1870659584 >> 3 != (signed)((unsigned)-1870659584 >> 3) || + -1870659584 >> 4 != (signed)((unsigned)-1870659584 >> 4) || + -1870659584 >> 8 != (signed)((unsigned)-1870659584 >> 8) || + -1870659584 >> 16 != (signed)((unsigned)-1870659584 >> 16) || + -1870659584 >> 24 != (signed)((unsigned)-1870659584 >> 24) || + -1870659584 >> 31 != (signed)((unsigned)-1870659584 >> 31) || + 0; + ]])], [ + success_logical=yes + ]) + + + AC_DEFINE([ARITHMETIC_RIGHT_SHIFT], 1, [Possible value for SIGNED_RIGHT_SHIFT_IS]) + AC_DEFINE([LOGICAL_RIGHT_SHIFT], 2, [Possible value for SIGNED_RIGHT_SHIFT_IS]) + AC_DEFINE([UNKNOWN_RIGHT_SHIFT], 3, [Possible value for SIGNED_RIGHT_SHIFT_IS]) + + if test "$success_arithmetic" = "yes" && test "$success_logica" = "yes" ; then + AC_MSG_ERROR("Right shift appears to be both arithmetic and logical!") + elif test "$success_arithmetic" = "yes" ; then + ax_signed_right_shift=arithmetic + AC_DEFINE([SIGNED_RIGHT_SHIFT_IS], 1, + [Indicates the effect of the right shift operator + on negative signed integers]) + elif test "$success_logical" = "yes" ; then + ax_signed_right_shift=logical + AC_DEFINE([SIGNED_RIGHT_SHIFT_IS], 2, + [Indicates the effect of the right shift operator + on negative signed integers]) + else + ax_signed_right_shift=unknown + AC_DEFINE([SIGNED_RIGHT_SHIFT_IS], 3, + [Indicates the effect of the right shift operator + on negative signed integers]) + fi + + AC_MSG_RESULT($ax_signed_right_shift) + ]) diff --git a/aclocal/ax_thrift_internal.m4 b/aclocal/ax_thrift_internal.m4 new file mode 100644 index 00000000..979bec6b --- /dev/null +++ b/aclocal/ax_thrift_internal.m4 @@ -0,0 +1,39 @@ +dnl @synopsis AX_THRIFT_GEN(SHORT_LANGUAGE, LONG_LANGUAGE, DEFAULT) +dnl @synopsis AX_THRIFT_LIB(SHORT_LANGUAGE, LONG_LANGUAGE, DEFAULT) +dnl +dnl Allow a particular language generator to be disabled. +dnl Allow a particular language library to be disabled. +dnl +dnl These macros have poor error handling and are poorly documented. +dnl They are intended only for internal use by the Thrift compiler. +dnl +dnl @version 2008-02-20 +dnl @license AllPermissive +dnl +dnl Copyright (C) 2009 David Reiss +dnl Copying and distribution of this file, with or without modification, +dnl are permitted in any medium without royalty provided the copyright +dnl notice and this notice are preserved. + +AC_DEFUN([AX_THRIFT_GEN], + [ + AC_ARG_ENABLE([gen-$1], + AC_HELP_STRING([--enable-gen-$1], [enable the $2 compiler @<:@default=$3@:>@]), + [ax_thrift_gen_$1="$enableval"], + [ax_thrift_gen_$1=$3] + ) + dnl I'd like to run the AM_CONDITIONAL here, but automake likes + dnl all AM_CONDITIONALs to be nice and explicit in configure.ac. + dnl AM_CONDITIONAL([THRIFT_GEN_$1], [test "$ax_thrift_gen_$1" = "yes"]) + ]) + +AC_DEFUN([AX_THRIFT_LIB], + [ + AC_ARG_WITH($1, + AC_HELP_STRING([--with-$1], [build the $2 library @<:@default=$3@:>@]), + [with_$1="$withval"], + [with_$1=$3] + ) + dnl What we do here is going to vary from library to library, + dnl so we can't really generalize (yet!). + ]) diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 00000000..45ac8d52 --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +./cleanup.sh + +autoscan || exit 1 +aclocal -I ./aclocal || exit 1 +autoheader || exit 1 + +if libtoolize --version 1 >/dev/null 2>/dev/null; then + libtoolize --automake || exit 1 +elif glibtoolize --version 1 >/dev/null 2>/dev/null; then + glibtoolize --automake || exit 1 +fi + +autoconf +automake -ac --add-missing --foreign || exit 1 diff --git a/cleanup.sh b/cleanup.sh new file mode 100755 index 00000000..0ea55bb9 --- /dev/null +++ b/cleanup.sh @@ -0,0 +1,58 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +topsrcdir="`dirname $0`" +cd "$topsrcdir" + +make -k clean >/dev/null 2>&1 +make -k distclean >/dev/null 2>&1 +find . -name Makefile.in -exec rm -f {} \; +rm -rf \ +AUTHORS \ +ChangeLog \ +INSTALL \ +Makefile \ +Makefile.in \ +Makefile.orig \ +aclocal.m4 \ +autom4te.cache \ +autoscan.log \ +config.guess \ +config.h \ +config.hin \ +config.hin~ \ +config.log \ +config.status \ +config.status.lineno \ +config.sub \ +configure \ +configure.lineno \ +configure.scan \ +depcomp \ +.deps \ +install-sh \ +.libs \ +libtool \ +ltmain.sh \ +missing \ +ylwrap \ +if/gen-* \ +test/gen-* diff --git a/compiler/cpp/Makefile.am b/compiler/cpp/Makefile.am new file mode 100644 index 00000000..3838facf --- /dev/null +++ b/compiler/cpp/Makefile.am @@ -0,0 +1,136 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +AM_YFLAGS = -d +BUILT_SOURCES = + +bin_PROGRAMS = thrift + +thrift_OBJDIR = obj + +thrift_SOURCES = src/thrifty.yy \ + src/thriftl.ll \ + src/main.cc \ + src/md5.c \ + src/generate/t_generator.cc \ + src/globals.h \ + src/main.h \ + src/platform.h \ + src/md5.h \ + src/parse/t_doc.h \ + src/parse/t_type.h \ + src/parse/t_base_type.h \ + src/parse/t_enum.h \ + src/parse/t_enum_value.h \ + src/parse/t_typedef.h \ + src/parse/t_container.h \ + src/parse/t_list.h \ + src/parse/t_set.h \ + src/parse/t_map.h \ + src/parse/t_struct.h \ + src/parse/t_field.h \ + src/parse/t_service.h \ + src/parse/t_function.h \ + src/parse/t_program.h \ + src/parse/t_scope.h \ + src/parse/t_const.h \ + src/parse/t_const_value.h \ + src/generate/t_generator.h \ + src/generate/t_oop_generator.h + +if THRIFT_GEN_cpp +thrift_SOURCES += src/generate/t_cpp_generator.cc +endif +if THRIFT_GEN_java +thrift_SOURCES += src/generate/t_java_generator.cc +endif +if THRIFT_GEN_csharp +thrift_SOURCES += src/generate/t_csharp_generator.cc +endif +if THRIFT_GEN_py +thrift_SOURCES += src/generate/t_py_generator.cc +endif +if THRIFT_GEN_rb +thrift_SOURCES += src/generate/t_rb_generator.cc +endif +if THRIFT_GEN_perl +thrift_SOURCES += src/generate/t_perl_generator.cc +endif +if THRIFT_GEN_php +thrift_SOURCES += src/generate/t_php_generator.cc +endif +if THRIFT_GEN_erl +thrift_SOURCES += src/generate/t_erl_generator.cc +endif +if THRIFT_GEN_cocoa +thrift_SOURCES += src/generate/t_cocoa_generator.cc +endif +if THRIFT_GEN_st +thrift_SOURCES += src/generate/t_st_generator.cc +endif +if THRIFT_GEN_ocaml +thrift_SOURCES += src/generate/t_ocaml_generator.cc +endif +if THRIFT_GEN_hs +thrift_SOURCES += src/generate/t_hs_generator.cc +endif +if THRIFT_GEN_xsd +thrift_SOURCES += src/generate/t_xsd_generator.cc +endif +if THRIFT_GEN_html +thrift_SOURCES += src/generate/t_html_generator.cc +endif + +thrift_CXXFLAGS = -Wall -I$(srcdir)/src $(BOOST_CPPFLAGS) +thrift_LDFLAGS = -Wall $(BOOST_LDFLAGS) + +thrift_LDADD = @LEXLIB@ + +EXTRA_DIST = README + +clean-local: + $(RM) thriftl.cc thrifty.cc thrifty.h version.h + +src/main.cc: version.h + +# Adding this to BUILT_SOURCES will cause version.h to be +# regenerated on every "make all" or "make check", which is +# necessary because it changes whenever we "svn up" or similar. +# Ideally, we would like this to be regenerated whenever the +# compiler is rebuilt, but every way we could think of to do +# that caused unnecessary rebuilds of the compiler. +BUILT_SOURCES += regen_version_h + +THRIFT_VERSION=$(shell /bin/sh $(top_srcdir)/print_version.sh -v) +THRIFT_REVISION=$(shell /bin/sh $(top_srcdir)/print_version.sh -r) + +regen_version_h: + @printf "Regenerating version.h... " + @TMPFILE=`mktemp ./version_h.tmp_XXXXXX` ; \ + echo "// AUTOGENERATED, DO NOT EDIT" > $$TMPFILE ; \ + echo '#define THRIFT_VERSION "$(THRIFT_VERSION)"' >> $$TMPFILE ; \ + echo '#define THRIFT_REVISION "$(THRIFT_REVISION)"' >> $$TMPFILE ; \ + if cmp $$TMPFILE version.h >/dev/null ; \ + then \ + rm -f $$TMPFILE ; \ + echo "No changes." ; \ + else \ + mv $$TMPFILE version.h ; \ + echo "Updated." ; \ + fi diff --git a/compiler/cpp/README b/compiler/cpp/README new file mode 100644 index 00000000..fb100a82 --- /dev/null +++ b/compiler/cpp/README @@ -0,0 +1,39 @@ +Thrift Code Compiler + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Thrift Code Compiler +==================== + +This compiler takes thrift files as input and generates output code across +various programming languages. To build and install it, do this: + + ./bootstrap.sh + ./configure + make + sudo make install + +It requires some form of LEX and YACC to be installed, which should be +picked up by autoconf. + +Not much else to report here. You'll have to look at the code to get your +questions answered. Or just run the executable after you build and take +a look at the usage message. diff --git a/compiler/cpp/src/generate/t_cocoa_generator.cc b/compiler/cpp/src/generate/t_cocoa_generator.cc new file mode 100644 index 00000000..48c853c2 --- /dev/null +++ b/compiler/cpp/src/generate/t_cocoa_generator.cc @@ -0,0 +1,2059 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include "t_oop_generator.h" +#include "platform.h" +using namespace std; + + +/** + * Objective-C code generator. + * + * mostly copy/pasting/tweaking from mcslee's work. + */ +class t_cocoa_generator : public t_oop_generator { + public: + t_cocoa_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_oop_generator(program) + { + out_dir_base_ = "gen-cocoa"; + } + + /** + * Init and close methods + */ + + void init_generator(); + void close_generator(); + + void generate_consts(std::vector consts); + + /** + * Program-level generation functions + */ + + void generate_typedef (t_typedef* ttypedef); + void generate_enum (t_enum* tenum); + void generate_struct (t_struct* tstruct); + void generate_xception(t_struct* txception); + void generate_service (t_service* tservice); + + void print_const_value(std::ofstream& out, std::string name, t_type* type, t_const_value* value); + std::string render_const_value(std::string name, t_type* type, t_const_value* value, + bool containerize_it=false); + + void generate_cocoa_struct(t_struct* tstruct, bool is_exception); + void generate_cocoa_struct_interface(std::ofstream& out, t_struct* tstruct, bool is_xception=false); + void generate_cocoa_struct_implementation(std::ofstream& out, t_struct* tstruct, bool is_xception=false, bool is_result=false); + void generate_cocoa_struct_initializer_signature(std::ofstream& out, + t_struct* tstruct); + void generate_cocoa_struct_field_accessor_declarations(std::ofstream& out, + t_struct* tstruct, + bool is_exception); + void generate_cocoa_struct_field_accessor_implementations(std::ofstream& out, + t_struct* tstruct, + bool is_exception); + void generate_cocoa_struct_reader(std::ofstream& out, t_struct* tstruct); + void generate_cocoa_struct_result_writer(std::ofstream& out, t_struct* tstruct); + void generate_cocoa_struct_writer(std::ofstream& out, t_struct* tstruct); + void generate_cocoa_struct_description(std::ofstream& out, t_struct* tstruct); + + std::string function_result_helper_struct_type(t_function* tfunction); + void generate_function_helpers(t_function* tfunction); + + /** + * Service-level generation functions + */ + + void generate_cocoa_service_protocol (std::ofstream& out, t_service* tservice); + void generate_cocoa_service_client_interface (std::ofstream& out, t_service* tservice); + void generate_cocoa_service_client_implementation (std::ofstream& out, t_service* tservice); + void generate_cocoa_service_helpers (t_service* tservice); + void generate_service_client (t_service* tservice); + void generate_service_server (t_service* tservice); + void generate_process_function (t_service* tservice, t_function* tfunction); + + /** + * Serialization constructs + */ + + void generate_deserialize_field (std::ofstream& out, + t_field* tfield, + std::string fieldName); + + void generate_deserialize_struct (std::ofstream& out, + t_struct* tstruct, + std::string prefix=""); + + void generate_deserialize_container (std::ofstream& out, + t_type* ttype, + std::string prefix=""); + + void generate_deserialize_set_element (std::ofstream& out, + t_set* tset, + std::string prefix=""); + + void generate_deserialize_map_element (std::ofstream& out, + t_map* tmap, + std::string prefix=""); + + void generate_deserialize_list_element (std::ofstream& out, + t_list* tlist, + std::string prefix=""); + + void generate_serialize_field (std::ofstream& out, + t_field* tfield, + std::string prefix=""); + + void generate_serialize_struct (std::ofstream& out, + t_struct* tstruct, + std::string fieldName=""); + + void generate_serialize_container (std::ofstream& out, + t_type* ttype, + std::string prefix=""); + + void generate_serialize_map_element (std::ofstream& out, + t_map* tmap, + std::string iter, + std::string map); + + void generate_serialize_set_element (std::ofstream& out, + t_set* tmap, + std::string iter); + + void generate_serialize_list_element (std::ofstream& out, + t_list* tlist, + std::string index, + std::string listName); + + /** + * Helper rendering functions + */ + + std::string cocoa_prefix(); + std::string cocoa_imports(); + std::string cocoa_thrift_imports(); + std::string type_name(t_type* ttype, bool class_ref=false); + std::string base_type_name(t_base_type* tbase); + std::string declare_field(t_field* tfield); + std::string function_signature(t_function* tfunction); + std::string argument_list(t_struct* tstruct); + std::string type_to_enum(t_type* ttype); + std::string format_string_for_type(t_type* type); + std::string call_field_setter(t_field* tfield, std::string fieldName); + std::string containerize(t_type * ttype, std::string fieldName); + std::string decontainerize(t_field * tfield, std::string fieldName); + + bool type_can_be_null(t_type* ttype) { + ttype = get_true_type(ttype); + + return + ttype->is_container() || + ttype->is_struct() || + ttype->is_xception() || + ttype->is_string(); + } + + private: + + std::string cocoa_prefix_; + std::string constants_declarations_; + + /** + * File streams + */ + + std::ofstream f_header_; + std::ofstream f_impl_; + +}; + + +/** + * Prepares for file generation by opening up the necessary file output + * streams. + */ +void t_cocoa_generator::init_generator() { + // Make output directory + MKDIR(get_out_dir().c_str()); + cocoa_prefix_ = program_->get_namespace("cocoa"); + + // we have a .h header file... + string f_header_name = program_name_+".h"; + string f_header_fullname = get_out_dir()+f_header_name; + f_header_.open(f_header_fullname.c_str()); + + f_header_ << + autogen_comment() << + endl; + + f_header_ << + cocoa_imports() << + cocoa_thrift_imports(); + + // ...and a .m implementation file + string f_impl_name = get_out_dir()+program_name_+".m"; + f_impl_.open(f_impl_name.c_str()); + + f_impl_ << + autogen_comment() << + endl; + + f_impl_ << + cocoa_imports() << + cocoa_thrift_imports() << + "#import \"" << f_header_name << "\"" << endl << + endl; + +} + +/** + * Prints standard Cocoa imports + * + * @return List of imports for Cocoa libraries + */ +string t_cocoa_generator::cocoa_imports() { + return + string() + + "#import \n" + + "\n"; +} + +/** + * Prints thrift runtime imports + * + * @return List of imports necessary for thrift runtime + */ +string t_cocoa_generator::cocoa_thrift_imports() { + string result = string() + + "#import \n" + + "#import \n" + + "#import \n" + + "\n"; + + // Include other Thrift includes + const vector& includes = program_->get_includes(); + for (size_t i = 0; i < includes.size(); ++i) { + result += "#import \"" + includes[i]->get_name() + ".h\"" + "\n"; + } + result += "\n"; + + return result; +} + + +/** + * Finish up generation. + */ +void t_cocoa_generator::close_generator() +{ + // stick our constants declarations at the end of the header file + // since they refer to things we are defining. + f_header_ << constants_declarations_ << endl; +} + +/** + * Generates a typedef. This is just a simple 1-liner in objective-c + * + * @param ttypedef The type definition + */ +void t_cocoa_generator::generate_typedef(t_typedef* ttypedef) { + f_header_ << + indent() << "typedef " << type_name(ttypedef->get_type()) << " " << cocoa_prefix_ << ttypedef->get_symbolic() << ";" << endl << + endl; +} + +/** + * Generates code for an enumerated type. In Objective-C, this is + * essentially the same as the thrift definition itself, using the + * enum keyword in Objective-C. For namespace purposes, the name of + * the enum plus an underscore is prefixed onto each element. + * + * @param tenum The enumeration + */ +void t_cocoa_generator::generate_enum(t_enum* tenum) { + f_header_ << + indent() << "enum " << cocoa_prefix_ << tenum->get_name() << " {" << endl; + indent_up(); + + vector constants = tenum->get_constants(); + vector::iterator c_iter; + bool first = true; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if (first) { + first = false; + } else { + f_header_ << + "," << endl; + } + f_header_ << + indent() << tenum->get_name() << "_" << (*c_iter)->get_name(); + if ((*c_iter)->has_value()) { + f_header_ << + " = " << (*c_iter)->get_value(); + } + } + + indent_down(); + f_header_ << + endl << + "};" << endl << + endl; +} + +/** + * Generates a class that holds all the constants. Primitive values + * could have been placed outside this class, but I just put + * everything in for consistency. + */ +void t_cocoa_generator::generate_consts(std::vector consts) { + std::ostringstream const_interface; + string constants_class_name = cocoa_prefix_ + program_name_ + "Constants"; + + const_interface << "@interface " << constants_class_name << " "; + scope_up(const_interface); + scope_down(const_interface); + + // getter method for each constant defined. + vector::iterator c_iter; + for (c_iter = consts.begin(); c_iter != consts.end(); ++c_iter) { + string name = (*c_iter)->get_name(); + t_type* type = (*c_iter)->get_type(); + const_interface << + "+ (" << type_name(type) << ") " << name << ";" << endl; + } + + const_interface << "@end"; + + // this gets spit into the header file in ::close_generator + constants_declarations_ = const_interface.str(); + + // static variables in the .m hold all constant values + for (c_iter = consts.begin(); c_iter != consts.end(); ++c_iter) { + string name = (*c_iter)->get_name(); + t_type* type = (*c_iter)->get_type(); + f_impl_ << + "static " << type_name(type) << " " << cocoa_prefix_ << name; + if (!type->is_container() && !type->is_struct()) { + f_impl_ << " = " << render_const_value(name, type, (*c_iter)->get_value()); + } + f_impl_ << ";" << endl; + } + f_impl_ << endl; + + f_impl_ << "@implementation " << constants_class_name << endl; + + // initialize complex constants when the class is loaded + f_impl_ << "+ (void) initialize "; + scope_up(f_impl_); + + for (c_iter = consts.begin(); c_iter != consts.end(); ++c_iter) { + if ((*c_iter)->get_type()->is_container() || + (*c_iter)->get_type()->is_struct()) { + string name = (*c_iter)->get_name(); + f_impl_ << indent() << name << " = " << render_const_value(name, + (*c_iter)->get_type(), + (*c_iter)->get_value()); + f_impl_ << ";" << endl; + } + } + scope_down(f_impl_); + + // getter method for each constant + for (c_iter = consts.begin(); c_iter != consts.end(); ++c_iter) { + string name = (*c_iter)->get_name(); + t_type* type = (*c_iter)->get_type(); + f_impl_ << + "+ (" << type_name(type) << ") " << name; + scope_up(f_impl_); + indent(f_impl_) << "return " << name << ";" << endl; + scope_down(f_impl_); + } + + f_impl_ << "@end" << endl << endl; +} + + +/** + * Generates a struct definition for a thrift data type. This is a class + * with protected data members, read(), write(), and getters and setters. + * + * @param tstruct The struct definition + */ +void t_cocoa_generator::generate_struct(t_struct* tstruct) { + generate_cocoa_struct_interface(f_header_, tstruct, false); + generate_cocoa_struct_implementation(f_impl_, tstruct, false); +} + +/** + * Exceptions are structs, but they inherit from NSException + * + * @param tstruct The struct definition + */ +void t_cocoa_generator::generate_xception(t_struct* txception) { + generate_cocoa_struct_interface(f_header_, txception, true); + generate_cocoa_struct_implementation(f_impl_, txception, true); +} + + +/** + * Generate the interface for a struct + * + * @param tstruct The struct definition + */ +void t_cocoa_generator::generate_cocoa_struct_interface(ofstream &out, + t_struct* tstruct, + bool is_exception) { + out << "@interface " << cocoa_prefix_ << tstruct->get_name() << " : "; + + if (is_exception) { + out << "NSException "; + } else { + out << "NSObject "; + } + + scope_up(out); + + // members are protected. this is redundant, but explicit. + // f_header_ << endl << "@protected:" << endl; + + const vector& members = tstruct->get_members(); + + // member varialbes + vector::const_iterator m_iter; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + out << indent() << declare_field(*m_iter) << endl; + } + + if (members.size() > 0) { + out << endl; + // isset fields + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + indent(out) << + "BOOL __" << (*m_iter)->get_name() << "_isset;" << endl; + } + } + + scope_down(out); + out << endl; + + // initializer for all fields + if (!members.empty()) { + generate_cocoa_struct_initializer_signature(out, tstruct); + out << ";" << endl; + } + out << endl; + + // read and write + out << "- (void) read: (id ) inProtocol;" << endl; + out << "- (void) write: (id ) outProtocol;" << endl; + out << endl; + + // getters and setters + generate_cocoa_struct_field_accessor_declarations(out, tstruct, is_exception); + + out << "@end" << endl << endl; +} + + +/** + * Generate signature for initializer of struct with a parameter for + * each field. + */ +void t_cocoa_generator::generate_cocoa_struct_initializer_signature(ofstream &out, + t_struct* tstruct) { + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + indent(out) << "- (id) initWith"; + for (m_iter = members.begin(); m_iter != members.end(); ) { + if (m_iter == members.begin()) { + out << capitalize((*m_iter)->get_name()); + } else { + out << (*m_iter)->get_name(); + } + out << ": (" << type_name((*m_iter)->get_type()) << ") " << + (*m_iter)->get_name(); + ++m_iter; + if (m_iter != members.end()) { + out << " "; + } + } +} + + +/** + * Generate getter and setter declarations for all fields, plus an + * IsSet getter. + */ +void t_cocoa_generator::generate_cocoa_struct_field_accessor_declarations(ofstream &out, + t_struct* tstruct, + bool is_exception) { + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + out << indent() << "- (" << type_name((*m_iter)->get_type()) << ") " << decapitalize((*m_iter)->get_name()) << ";" << endl; + out << indent() << "- (void) set" << capitalize((*m_iter)->get_name()) << + ": (" << type_name((*m_iter)->get_type()) << ") " << (*m_iter)->get_name() << ";" << endl; + out << indent() << "- (BOOL) " << (*m_iter)->get_name() << "IsSet;" << endl << endl; + } +} + + +/** + * Generate struct implementation. + * + * @param tstruct The struct definition + * @param is_exception Is this an exception? + * @param is_result If this is a result it needs a different writer + */ +void t_cocoa_generator::generate_cocoa_struct_implementation(ofstream &out, + t_struct* tstruct, + bool is_exception, + bool is_result) { + indent(out) << + "@implementation " << cocoa_prefix_ << tstruct->get_name() << endl; + + // exceptions need to call the designated initializer on NSException + if (is_exception) { + out << indent() << "- (id) init" << endl; + scope_up(out); + out << indent() << "return [super initWithName: @\"" << tstruct->get_name() << + "\" reason: @\"unknown\" userInfo: nil];" << endl; + scope_down(out); + } + + // initializer with all fields as params + const vector& members = tstruct->get_members(); + if (!members.empty()) { + generate_cocoa_struct_initializer_signature(out, tstruct); + out << endl; + scope_up(out); + if (is_exception) { + out << indent() << "self = [self init];" << endl; + } else { + out << indent() << "self = [super init];" << endl; + } + + vector::const_iterator m_iter; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + t_type* t = get_true_type((*m_iter)->get_type()); + out << indent() << "__" << (*m_iter)->get_name() << " = "; + if (type_can_be_null(t)) { + out << "[" << (*m_iter)->get_name() << " retain];" << endl; + } else { + out << (*m_iter)->get_name() << ";" << endl; + } + out << indent() << "__" << (*m_iter)->get_name() << "_isset = YES;" << endl; + } + + out << indent() << "return self;" << endl; + scope_down(out); + out << endl; + } + + // dealloc + if (!members.empty()) { + out << "- (void) dealloc" << endl; + scope_up(out); + + vector::const_iterator m_iter; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + t_type* t = get_true_type((*m_iter)->get_type()); + if (type_can_be_null(t)) { + indent(out) << "[__" << (*m_iter)->get_name() << " release];" << endl; + } + } + + out << indent() << "[super dealloc];" << endl; + scope_down(out); + out << endl; + } + + // the rest of the methods + generate_cocoa_struct_field_accessor_implementations(out, tstruct, is_exception); + generate_cocoa_struct_reader(out, tstruct); + if (is_result) { + generate_cocoa_struct_result_writer(out, tstruct); + } else { + generate_cocoa_struct_writer(out, tstruct); + } + generate_cocoa_struct_description(out, tstruct); + + out << "@end" << endl << endl; +} + + +/** + * Generates a function to read all the fields of the struct. + * + * @param tstruct The struct definition + */ +void t_cocoa_generator::generate_cocoa_struct_reader(ofstream& out, + t_struct* tstruct) { + out << + "- (void) read: (id ) inProtocol" << endl; + scope_up(out); + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + // Declare stack tmp variables + indent(out) << "NSString * fieldName;" << endl; + indent(out) << "int fieldType;" << endl; + indent(out) << "int fieldID;" << endl; + out << endl; + + indent(out) << "[inProtocol readStructBeginReturningName: NULL];" << endl; + + // Loop over reading in fields + indent(out) << + "while (true)" << endl; + scope_up(out); + + // Read beginning field marker + indent(out) << + "[inProtocol readFieldBeginReturningName: &fieldName type: &fieldType fieldID: &fieldID];" << endl; + + // Check for field STOP marker and break + indent(out) << + "if (fieldType == TType_STOP) { " << endl; + indent_up(); + indent(out) << + "break;" << endl; + indent_down(); + indent(out) << + "}" << endl; + + // Switch statement on the field we are reading + indent(out) << + "switch (fieldID)" << endl; + + scope_up(out); + + // Generate deserialization code for known cases + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + indent(out) << + "case " << (*f_iter)->get_key() << ":" << endl; + indent_up(); + indent(out) << + "if (fieldType == " << type_to_enum((*f_iter)->get_type()) << ") {" << endl; + indent_up(); + + generate_deserialize_field(out, *f_iter, "fieldValue"); + indent(out) << call_field_setter(*f_iter, "fieldValue") << endl; + // if this is an allocated field, release it since the struct + // is now retaining it + if (type_can_be_null((*f_iter)->get_type())) { + // deserialized strings are autorelease, so don't release them + if (!(get_true_type((*f_iter)->get_type())->is_string())) { + indent(out) << "[fieldValue release];" << endl; + } + } + + indent_down(); + out << + indent() << "} else { " << endl << + indent() << " [TProtocolUtil skipType: fieldType onProtocol: inProtocol];" << endl << + indent() << "}" << endl << + indent() << "break;" << endl; + indent_down(); + } + + // In the default case we skip the field + out << + indent() << "default:" << endl << + indent() << " [TProtocolUtil skipType: fieldType onProtocol: inProtocol];" << endl << + indent() << " break;" << endl; + + scope_down(out); + + // Read field end marker + indent(out) << + "[inProtocol readFieldEnd];" << endl; + + scope_down(out); + + out << + indent() << "[inProtocol readStructEnd];" << endl; + + indent_down(); + out << + indent() << "}" << endl << + endl; +} + +/** + * Generates a function to write all the fields of the struct + * + * @param tstruct The struct definition + */ +void t_cocoa_generator::generate_cocoa_struct_writer(ofstream& out, + t_struct* tstruct) { + out << + indent() << "- (void) write: (id ) outProtocol {" << endl; + indent_up(); + + string name = tstruct->get_name(); + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + out << + indent() << "[outProtocol writeStructBeginWithName: @\"" << name << "\"];" << endl; + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + out << + indent() << "if (__" << (*f_iter)->get_name() << "_isset) {" << endl; + indent_up(); + bool null_allowed = type_can_be_null((*f_iter)->get_type()); + if (null_allowed) { + out << + indent() << "if (__" << (*f_iter)->get_name() << " != nil) {" << endl; + indent_up(); + } + + indent(out) << "[outProtocol writeFieldBeginWithName: @\"" << + (*f_iter)->get_name() << "\" type: " << type_to_enum((*f_iter)->get_type()) << + " fieldID: " << (*f_iter)->get_key() << "];" << endl; + + // Write field contents + generate_serialize_field(out, *f_iter, "__"+(*f_iter)->get_name()); + + // Write field closer + indent(out) << + "[outProtocol writeFieldEnd];" << endl; + + if (null_allowed) { + scope_down(out); + } + scope_down(out); + } + // Write the struct map + out << + indent() << "[outProtocol writeFieldStop];" << endl << + indent() << "[outProtocol writeStructEnd];" << endl; + + indent_down(); + out << + indent() << "}" << endl << + endl; +} + +/** + * Generates a function to write all the fields of the struct, which + * is a function result. These fields are only written if they are + * set, and only one of them can be set at a time. + * + * @param tstruct The struct definition + */ +void t_cocoa_generator::generate_cocoa_struct_result_writer(ofstream& out, + t_struct* tstruct) { + out << + indent() << "- (void) write: (id ) outProtocol {" << endl; + indent_up(); + + string name = tstruct->get_name(); + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + out << + indent() << "[outProtocol writeStructBeginWithName: @\"" << name << "\"];" << endl; + + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + out << + endl << + indent() << "if "; + } else { + out << + " else if "; + } + + out << + "(__" << (*f_iter)->get_name() << "_isset) {" << endl; + indent_up(); + + bool null_allowed = type_can_be_null((*f_iter)->get_type()); + if (null_allowed) { + out << + indent() << "if (__" << (*f_iter)->get_name() << " != nil) {" << endl; + indent_up(); + } + + indent(out) << "[outProtocol writeFieldBeginWithName: @\"" << + (*f_iter)->get_name() << "\" type: " << type_to_enum((*f_iter)->get_type()) << + " fieldID: " << (*f_iter)->get_key() << "];" << endl; + + // Write field contents + generate_serialize_field(out, *f_iter, "__"+(*f_iter)->get_name()); + + // Write field closer + indent(out) << + "[outProtocol writeFieldEnd];" << endl; + + if (null_allowed) { + indent_down(); + indent(out) << "}" << endl; + } + + indent_down(); + indent(out) << "}"; + } + // Write the struct map + out << + endl << + indent() << "[outProtocol writeFieldStop];" << endl << + indent() << "[outProtocol writeStructEnd];" << endl; + + indent_down(); + out << + indent() << "}" << endl << + endl; +} + +/** + * Generate property accessor methods for all fields in the struct. + * getter, setter, isset getter. + * + * @param tstruct The struct definition + */ +void t_cocoa_generator::generate_cocoa_struct_field_accessor_implementations(ofstream& out, + t_struct* tstruct, + bool is_exception) { + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + t_field* field = *f_iter; + t_type* type = get_true_type(field->get_type()); + std::string field_name = field->get_name(); + std::string cap_name = field_name; + cap_name[0] = toupper(cap_name[0]); + + // Simple getter + indent(out) << "- (" << type_name(type) << ") "; + out << field_name << " {" << endl; + indent_up(); + if (!type_can_be_null(type)) { + indent(out) << "return __" << field_name << ";" << endl; + } else { + indent(out) << "return [[__" << field_name << " retain] autorelease];" << endl; + } + indent_down(); + indent(out) << "}" << endl << endl; + + // Simple setter + indent(out) << "- (void) set" << cap_name << ": (" << type_name(type) << + ") " << field_name << " {" << endl; + indent_up(); + if (!type_can_be_null(type)) { + indent(out) << "__" << field_name << " = " << field_name << ";" << endl; + } else { + indent(out) << "[" << field_name << " retain];" << endl; + indent(out) << "[__" << field_name << " release];" << endl; + indent(out) << "__" << field_name << " = " << field_name << ";" << endl; + } + indent(out) << "__" << field_name << "_isset = YES;" << endl; + indent_down(); + indent(out) << "}" << endl << endl; + + // IsSet + indent(out) << "- (BOOL) " << field_name << "IsSet {" << endl; + indent_up(); + indent(out) << "return __" << field_name << "_isset;" << endl; + indent_down(); + indent(out) << "}" << endl << endl; + + // Unsetter - do we need this? + indent(out) << "- (void) unset" << cap_name << " {" << endl; + indent_up(); + if (type_can_be_null(type)) { + indent(out) << "[__" << field_name << " release];" << endl; + indent(out) << "__" << field_name << " = nil;" << endl; + } + indent(out) << "__" << field_name << "_isset = NO;" << endl; + indent_down(); + indent(out) << "}" << endl << endl; + } +} + +/** + * Generates a description method for the given struct + * + * @param tstruct The struct definition + */ +void t_cocoa_generator::generate_cocoa_struct_description(ofstream& out, + t_struct* tstruct) { + out << + indent() << "- (NSString *) description {" << endl; + indent_up(); + + out << + indent() << "NSMutableString * ms = [NSMutableString stringWithString: @\"" << + tstruct->get_name() << "(\"];" << endl; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + indent(out) << "[ms appendString: @\"" << (*f_iter)->get_name() << ":\"];" << endl; + } else { + indent(out) << "[ms appendString: @\"," << (*f_iter)->get_name() << ":\"];" << endl; + } + t_type* ttype = (*f_iter)->get_type(); + indent(out) << "[ms appendFormat: @\"" << format_string_for_type(ttype) << "\", __" << + (*f_iter)->get_name() << "];" << endl; + } + out << + indent() << "[ms appendString: @\")\"];" << endl << + indent() << "return [ms copy];" << endl; + + indent_down(); + indent(out) << "}" << endl << + endl; +} + + +/** + * Generates a thrift service. In Objective-C this consists of a + * protocol definition, a client interface and a client implementation. + * + * @param tservice The service definition + */ +void t_cocoa_generator::generate_service(t_service* tservice) { + generate_cocoa_service_protocol(f_header_, tservice); + generate_cocoa_service_client_interface(f_header_, tservice); + generate_cocoa_service_helpers(tservice); + generate_cocoa_service_client_implementation(f_impl_, tservice); +} + + +/** + * Generates structs for all the service return types + * + * @param tservice The service + */ +void t_cocoa_generator::generate_cocoa_service_helpers(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + generate_function_helpers(*f_iter); + } +} + +string t_cocoa_generator::function_result_helper_struct_type(t_function* tfunction) { + return capitalize(tfunction->get_name()) + "Result_"; +} + + +/** + * Generates a struct and helpers for a function. + * + * @param tfunction The function + */ +void t_cocoa_generator::generate_function_helpers(t_function* tfunction) { + if (tfunction->is_oneway()) { + return; + } + + // create a result struct with a success field of the return type, + // and a field for each type of exception thrown + t_struct result(program_, function_result_helper_struct_type(tfunction)); + t_field success(tfunction->get_returntype(), "success", 0); + if (!tfunction->get_returntype()->is_void()) { + result.append(&success); + } + + t_struct* xs = tfunction->get_xceptions(); + const vector& fields = xs->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + result.append(*f_iter); + } + + // generate the result struct + generate_cocoa_struct_interface(f_impl_, &result, false); + generate_cocoa_struct_implementation(f_impl_, &result, false, true); +} + +/** + * Generates a service protocol definition. + * + * @param tservice The service to generate a protocol definition for + */ +void t_cocoa_generator::generate_cocoa_service_protocol(ofstream& out, + t_service* tservice) { + out << "@protocol " << cocoa_prefix_ << tservice->get_name() << " " << endl; + + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + out << "- " << function_signature(*f_iter) << ";" << + " // throws "; + t_struct* xs = (*f_iter)->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + out << type_name((*x_iter)->get_type()) + ", "; + } + out << "TException" << endl; + } + out << "@end" << endl << endl; +} + + +/** + * Generates a service client interface definition. + * + * @param tservice The service to generate a client interface definition for + */ +void t_cocoa_generator::generate_cocoa_service_client_interface(ofstream& out, + t_service* tservice) { + out << "@interface " << cocoa_prefix_ << tservice->get_name() << "Client : NSObject <" << + cocoa_prefix_ << tservice->get_name() << "> "; + + scope_up(out); + out << indent() << "id inProtocol;" << endl; + out << indent() << "id outProtocol;" << endl; + scope_down(out); + + out << "- (id) initWithProtocol: (id ) protocol;" << endl; + out << "- (id) initWithInProtocol: (id ) inProtocol outProtocol: (id ) outProtocol;" << endl; + out << "@end" << endl << endl; +} + + +/** + * Generates a service client implementation. + * + * @param tservice The service to generate an implementation for + */ +void t_cocoa_generator::generate_cocoa_service_client_implementation(ofstream& out, + t_service* tservice) { + out << "@implementation " << cocoa_prefix_ << tservice->get_name() << "Client" << endl; + + // initializers + out << "- (id) initWithProtocol: (id ) protocol" << endl; + scope_up(out); + out << indent() << "return [self initWithInProtocol: protocol outProtocol: protocol];" << endl; + scope_down(out); + out << endl; + + out << "- (id) initWithInProtocol: (id ) anInProtocol outProtocol: (id ) anOutProtocol" << endl; + scope_up(out); + out << indent() << "[super init];" << endl; + out << indent() << "inProtocol = [anInProtocol retain];" << endl; + out << indent() << "outProtocol = [anOutProtocol retain];" << endl; + out << indent() << "return self;" << endl; + scope_down(out); + out << endl; + + // dealloc + out << "- (void) dealloc" << endl; + scope_up(out); + out << indent() << "[inProtocol release];" << endl; + out << indent() << "[outProtocol release];" << endl; + out << indent() << "[super dealloc];" << endl; + scope_down(out); + out << endl; + + // generate client method implementations + vector functions = tservice->get_functions(); + vector::const_iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string funname = (*f_iter)->get_name(); + + t_function send_function(g_type_void, + string("send_") + (*f_iter)->get_name(), + (*f_iter)->get_arglist()); + + string argsname = (*f_iter)->get_name() + "_args"; + + // Open function + indent(out) << + "- " << function_signature(&send_function) << endl; + scope_up(out); + + // Serialize the request + out << + indent() << "[outProtocol writeMessageBeginWithName: @\"" << funname << "\"" << + " type: TMessageType_CALL" << + " sequenceID: 0];" << endl; + + out << + indent() << "[outProtocol writeStructBeginWithName: @\"" << argsname << "\"];" << endl; + + // write out function parameters + t_struct* arg_struct = (*f_iter)->get_arglist(); + const vector& fields = arg_struct->get_members(); + vector::const_iterator fld_iter; + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + string fieldName = (*fld_iter)->get_name(); + if (type_can_be_null((*fld_iter)->get_type())) { + out << indent() << "if (" << fieldName << " != nil)"; + scope_up(out); + } + out << + indent() << "[outProtocol writeFieldBeginWithName: @\"" << fieldName << "\"" + " type: " << type_to_enum((*fld_iter)->get_type()) << + " fieldID: " << (*fld_iter)->get_key() << "];" << endl; + + generate_serialize_field(out, *fld_iter, fieldName); + + out << + indent() << "[outProtocol writeFieldEnd];" << endl; + + if (type_can_be_null((*fld_iter)->get_type())) { + scope_down(out); + } + } + + out << + indent() << "[outProtocol writeFieldStop];" << endl; + out << + indent() << "[outProtocol writeStructEnd];" << endl; + + out << + indent() << "[outProtocol writeMessageEnd];" << endl << + indent() << "[[outProtocol transport] flush];" << endl; + + scope_down(out); + out << endl; + + if (!(*f_iter)->is_oneway()) { + t_struct noargs(program_); + t_function recv_function((*f_iter)->get_returntype(), + string("recv_") + (*f_iter)->get_name(), + &noargs, + (*f_iter)->get_xceptions()); + // Open function + indent(out) << + "- " << function_signature(&recv_function) << endl; + scope_up(out); + + // TODO(mcslee): Message validation here, was the seqid etc ok? + + // check for an exception + out << + indent() << "int msgType = 0;" << endl << + indent() << "[inProtocol readMessageBeginReturningName: nil type: &msgType sequenceID: NULL];" << endl << + indent() << "if (msgType == TMessageType_EXCEPTION) {" << endl << + indent() << " TApplicationException * x = [TApplicationException read: inProtocol];" << endl << + indent() << " [inProtocol readMessageEnd];" << endl << + indent() << " @throw x;" << endl << + indent() << "}" << endl; + + // FIXME - could optimize here to reduce creation of temporary objects. + string resultname = function_result_helper_struct_type(*f_iter); + out << + indent() << cocoa_prefix_ << resultname << " * result = [[[" << cocoa_prefix_ << + resultname << " alloc] init] autorelease];" << endl; + indent(out) << "[result read: inProtocol];" << endl; + indent(out) << "[inProtocol readMessageEnd];" << endl; + + // Careful, only return _result if not a void function + if (!(*f_iter)->get_returntype()->is_void()) { + out << + indent() << "if ([result successIsSet]) {" << endl << + indent() << " return [result success];" << endl << + indent() << "}" << endl; + } + + t_struct* xs = (*f_iter)->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + out << + indent() << "if ([result " << (*x_iter)->get_name() << "IsSet]) {" << endl << + indent() << " @throw [result " << (*x_iter)->get_name() << "];" << endl << + indent() << "}" << endl; + } + + // If you get here it's an exception, unless a void function + if ((*f_iter)->get_returntype()->is_void()) { + indent(out) << + "return;" << endl; + } else { + out << + indent() << "@throw [TApplicationException exceptionWithType: TApplicationException_MISSING_RESULT" << endl << + indent() << " reason: @\"" << (*f_iter)->get_name() << " failed: unknown result\"];" << endl; + } + + // Close function + scope_down(out); + out << endl; + } + + // Open function + indent(out) << + "- " << function_signature(*f_iter) << endl; + scope_up(out); + indent(out) << + "[self send_" << funname; + + // Declare the function arguments + bool first = true; + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + if (first) { + first = false; + } else { + out << " "; + } + out << ": " << (*fld_iter)->get_name(); + } + out << "];" << endl; + + if (!(*f_iter)->is_oneway()) { + out << indent(); + if (!(*f_iter)->get_returntype()->is_void()) { + out << "return "; + } + out << + "[self recv_" << funname << "];" << endl; + } + scope_down(out); + out << endl; + } + + indent_down(); + + out << "@end" << endl << endl; +} + + +/** + * Deserializes a field of any type. + * + * @param tfield The field + * @param fieldName The variable name for this field + */ +void t_cocoa_generator::generate_deserialize_field(ofstream& out, + t_field* tfield, + string fieldName) { + t_type* type = get_true_type(tfield->get_type()); + + if (type->is_void()) { + throw "CANNOT GENERATE DESERIALIZE CODE FOR void TYPE: " + + tfield->get_name(); + } + + if (type->is_struct() || type->is_xception()) { + generate_deserialize_struct(out, + (t_struct*)type, + fieldName); + } else if (type->is_container()) { + generate_deserialize_container(out, type, fieldName); + } else if (type->is_base_type() || type->is_enum()) { + indent(out) << + type_name(type) << " " << fieldName << " = [inProtocol "; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "compiler error: cannot serialize void field in a struct: " + + tfield->get_name(); + break; + case t_base_type::TYPE_STRING: + if (((t_base_type*)type)->is_binary()) { + out << "readBinary];"; + } else { + out << "readString];"; + } + break; + case t_base_type::TYPE_BOOL: + out << "readBool];"; + break; + case t_base_type::TYPE_BYTE: + out << "readByte];"; + break; + case t_base_type::TYPE_I16: + out << "readI16];"; + break; + case t_base_type::TYPE_I32: + out << "readI32];"; + break; + case t_base_type::TYPE_I64: + out << "readI64];"; + break; + case t_base_type::TYPE_DOUBLE: + out << "readDouble];"; + break; + default: + throw "compiler error: no Objective-C name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + out << "readI32];"; + } + out << + endl; + } else { + printf("DO NOT KNOW HOW TO DESERIALIZE FIELD '%s' TYPE '%s'\n", + tfield->get_name().c_str(), type_name(type).c_str()); + } +} + +/** + * Generates an unserializer for a struct, allocates the struct and invokes read: + */ +void t_cocoa_generator::generate_deserialize_struct(ofstream& out, + t_struct* tstruct, + string fieldName) { + indent(out) << type_name(tstruct) << fieldName << " = [[" << + type_name(tstruct, true) << " alloc] init];" << endl; + indent(out) << "[" << fieldName << " read: inProtocol];" << endl; +} + +/** + * Deserializes a container by reading its size and then iterating + */ +void t_cocoa_generator::generate_deserialize_container(ofstream& out, + t_type* ttype, + string fieldName) { + string size = tmp("_size"); + indent(out) << "int " << size << ";" << endl; + + // Declare variables, read header + if (ttype->is_map()) { + indent(out) + << "[inProtocol readMapBeginReturningKeyType: NULL valueType: NULL size: &" << + size << "];" << endl; + indent(out) << "NSMutableDictionary * " << fieldName << + " = [[NSMutableDictionary alloc] initWithCapacity: " << size << "];" << endl; + } else if (ttype->is_set()) { + indent(out) + << "[inProtocol readSetBeginReturningElementType: NULL size: &" << size << "];" << endl; + indent(out) << "NSMutableSet * " << fieldName << + " = [[NSMutableSet alloc] initWithCapacity: " << size << "];" << endl; + } else if (ttype->is_list()) { + indent(out) + << "[inProtocol readListBeginReturningElementType: NULL size: &" << size << "];" << endl; + indent(out) << "NSMutableArray * " << fieldName << + " = [[NSMutableArray alloc] initWithCapacity: " << size << "];" << endl; + } + // FIXME - the code above does not verify that the element types of + // the containers being read match the element types of the + // containers we are reading into. Does that matter? + + // For loop iterates over elements + string i = tmp("_i"); + indent(out) << "int " << i << ";" << endl << + indent() << "for (" << i << " = 0; " << + i << " < " << size << "; " << + "++" << i << ")" << endl; + + scope_up(out); + + if (ttype->is_map()) { + generate_deserialize_map_element(out, (t_map*)ttype, fieldName); + } else if (ttype->is_set()) { + generate_deserialize_set_element(out, (t_set*)ttype, fieldName); + } else if (ttype->is_list()) { + generate_deserialize_list_element(out, (t_list*)ttype, fieldName); + } + + scope_down(out); + + // Read container end + if (ttype->is_map()) { + indent(out) << "[inProtocol readMapEnd];" << endl; + } else if (ttype->is_set()) { + indent(out) << "[inProtocol readSetEnd];" << endl; + } else if (ttype->is_list()) { + indent(out) << "[inProtocol readListEnd];" << endl; + } + +} + + +/** + * Take a variable of a given type and wrap it in code to make it + * suitable for putting into a container, if necessary. Basically, + * wrap scaler primitives in NSNumber objects. + */ +string t_cocoa_generator::containerize(t_type * ttype, + string fieldName) +{ + // FIXME - optimize here to avoid autorelease pool? + ttype = get_true_type(ttype); + if (ttype->is_enum()) { + return "[NSNumber numberWithInt: " + fieldName + "]"; + } else if (ttype->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)ttype)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "can't containerize void"; + case t_base_type::TYPE_BOOL: + return "[NSNumber numberWithBool: " + fieldName + "]"; + case t_base_type::TYPE_BYTE: + return "[NSNumber numberWithUnsignedChar: " + fieldName + "]"; + case t_base_type::TYPE_I16: + return "[NSNumber numberWithShort: " + fieldName + "]"; + case t_base_type::TYPE_I32: + return "[NSNumber numberWithLong: " + fieldName + "]"; + case t_base_type::TYPE_I64: + return "[NSNumber numberWithLongLong: " + fieldName + "]"; + case t_base_type::TYPE_DOUBLE: + return "[NSNumber numberWithDouble: " + fieldName + "]"; + default: + break; + } + } + + // do nothing + return fieldName; +} + + +/** + * Generates code to deserialize a map element + */ +void t_cocoa_generator::generate_deserialize_map_element(ofstream& out, + t_map* tmap, + string fieldName) { + string key = tmp("_key"); + string val = tmp("_val"); + t_field fkey(tmap->get_key_type(), key); + t_field fval(tmap->get_val_type(), val); + + generate_deserialize_field(out, &fkey, key); + generate_deserialize_field(out, &fval, val); + + indent(out) << + "[" << fieldName << " setObject: " << containerize(fval.get_type(), val) << + " forKey: " << containerize(fkey.get_type(), key) << "];" << endl; +} + +/** + * Deserializes a set element + */ +void t_cocoa_generator::generate_deserialize_set_element(ofstream& out, + t_set* tset, + string fieldName) { + string elem = tmp("_elem"); + t_field felem(tset->get_elem_type(), elem); + + generate_deserialize_field(out, &felem, elem); + + indent(out) << + "[" << fieldName << " addObject: " << containerize(felem.get_type(), elem) << "];" << endl; +} + +/** + * Deserializes a list element + */ +void t_cocoa_generator::generate_deserialize_list_element(ofstream& out, + t_list* tlist, + string fieldName) { + string elem = tmp("_elem"); + t_field felem(tlist->get_elem_type(), elem); + + generate_deserialize_field(out, &felem, elem); + + indent(out) << + "[" << fieldName << " addObject: " << containerize(felem.get_type(), elem) << "];" << endl; +} + + +/** + * Serializes a field of any type. + * + * @param tfield The field to serialize + * @param fieldName Name to of the variable holding the field + */ +void t_cocoa_generator::generate_serialize_field(ofstream& out, + t_field* tfield, + string fieldName) { + t_type* type = get_true_type(tfield->get_type()); + + // Do nothing for void types + if (type->is_void()) { + throw "CANNOT GENERATE SERIALIZE CODE FOR void TYPE: " + + tfield->get_name(); + } + + if (type->is_struct() || type->is_xception()) { + generate_serialize_struct(out, + (t_struct*)type, + fieldName); + } else if (type->is_container()) { + generate_serialize_container(out, + type, + fieldName); + } else if (type->is_base_type() || type->is_enum()) { + indent(out) << + "[outProtocol "; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw + "compiler error: cannot serialize void field in a struct: " + fieldName; + break; + case t_base_type::TYPE_STRING: + if (((t_base_type*)type)->is_binary()) { + out << "writeBinary: " << fieldName << "];"; + } else { + out << "writeString: " << fieldName << "];"; + } + break; + case t_base_type::TYPE_BOOL: + out << "writeBool: " << fieldName << "];"; + break; + case t_base_type::TYPE_BYTE: + out << "writeByte: " << fieldName << "];"; + break; + case t_base_type::TYPE_I16: + out << "writeI16: " << fieldName << "];"; + break; + case t_base_type::TYPE_I32: + out << "writeI32: " << fieldName << "];"; + break; + case t_base_type::TYPE_I64: + out << "writeI64: " << fieldName << "];"; + break; + case t_base_type::TYPE_DOUBLE: + out << "writeDouble: " << fieldName << "];"; + break; + default: + throw "compiler error: no Java name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + out << "writeI32: " << fieldName << "];"; + } + out << endl; + } else { + printf("DO NOT KNOW HOW TO SERIALIZE FIELD '%s' TYPE '%s'\n", + tfield->get_name().c_str(), + type_name(type).c_str()); + } +} + +/** + * Serialize a struct. + * + * @param tstruct The struct to serialize + * @param fieldName Name of variable holding struct + */ +void t_cocoa_generator::generate_serialize_struct(ofstream& out, + t_struct* tstruct, + string fieldName) { + out << + indent() << "[" << fieldName << " write: outProtocol];" << endl; +} + +/** + * Serializes a container by writing its size then the elements. + * + * @param ttype The type of container + * @param fieldName Name of variable holding container + */ +void t_cocoa_generator::generate_serialize_container(ofstream& out, + t_type* ttype, + string fieldName) { + scope_up(out); + + if (ttype->is_map()) { + indent(out) << + "[outProtocol writeMapBeginWithKeyType: " << + type_to_enum(((t_map*)ttype)->get_key_type()) << " valueType: " << + type_to_enum(((t_map*)ttype)->get_val_type()) << " size: [" << + fieldName << " count]];" << endl; + } else if (ttype->is_set()) { + indent(out) << + "[outProtocol writeSetBeginWithElementType: " << + type_to_enum(((t_set*)ttype)->get_elem_type()) << " size: [" << + fieldName << " count]];" << endl; + } else if (ttype->is_list()) { + indent(out) << + "[outProtocol writeListBeginWithElementType: " << + type_to_enum(((t_list*)ttype)->get_elem_type()) << " size: [" << + fieldName << " count]];" << endl; + } + + string iter = tmp("_iter"); + string key; + if (ttype->is_map()) { + key = tmp("key"); + indent(out) << "NSEnumerator * " << iter << " = [" << fieldName << " keyEnumerator];" << endl; + indent(out) << "id " << key << ";" << endl; + indent(out) << "while ((" << key << " = [" << iter << " nextObject]))" << endl; + } else if (ttype->is_set()) { + key = tmp("obj"); + indent(out) << "NSEnumerator * " << iter << " = [" << fieldName << " objectEnumerator];" << endl; + indent(out) << "id " << key << ";" << endl; + indent(out) << "while ((" << key << " = [" << iter << " nextObject]))" << endl; + } else if (ttype->is_list()) { + key = tmp("i"); + indent(out) << "int " << key << ";" << endl; + indent(out) << + "for (" << key << " = 0; " << key << " < [" << fieldName << " count]; " << key << "++)" << endl; + } + + scope_up(out); + + if (ttype->is_map()) { + generate_serialize_map_element(out, (t_map*)ttype, key, fieldName); + } else if (ttype->is_set()) { + generate_serialize_set_element(out, (t_set*)ttype, key); + } else if (ttype->is_list()) { + generate_serialize_list_element(out, (t_list*)ttype, key, fieldName); + } + + scope_down(out); + + if (ttype->is_map()) { + indent(out) << + "[outProtocol writeMapEnd];" << endl; + } else if (ttype->is_set()) { + indent(out) << + "[outProtocol writeSetEnd];" << endl; + } else if (ttype->is_list()) { + indent(out) << + "[outProtocol writeListEnd];" << endl; + } + + scope_down(out); +} + +/** + * Given a field variable name, wrap it in code that converts it to a + * primitive type, if necessary. + */ +string t_cocoa_generator::decontainerize(t_field * tfield, + string fieldName) +{ + t_type * ttype = get_true_type(tfield->get_type()); + if (ttype->is_enum()) { + return "[" + fieldName + " intValue]"; + } else if (ttype->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)ttype)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "can't decontainerize void"; + case t_base_type::TYPE_BOOL: + return "[" + fieldName + " boolValue]"; + case t_base_type::TYPE_BYTE: + return "[" + fieldName + " unsignedCharValue]"; + case t_base_type::TYPE_I16: + return "[" + fieldName + " shortValue]"; + case t_base_type::TYPE_I32: + return "[" + fieldName + " longValue]"; + case t_base_type::TYPE_I64: + return "[" + fieldName + " longLongValue]"; + case t_base_type::TYPE_DOUBLE: + return "[" + fieldName + " doubleValue]"; + default: + break; + } + } + + // do nothing + return fieldName; +} + + +/** + * Serializes the members of a map. + */ +void t_cocoa_generator::generate_serialize_map_element(ofstream& out, + t_map* tmap, + string key, + string mapName) { + t_field kfield(tmap->get_key_type(), key); + generate_serialize_field(out, &kfield, decontainerize(&kfield, key)); + t_field vfield(tmap->get_val_type(), "[" + mapName + " objectForKey: " + key + "]"); + generate_serialize_field(out, &vfield, decontainerize(&vfield, vfield.get_name())); +} + +/** + * Serializes the members of a set. + */ +void t_cocoa_generator::generate_serialize_set_element(ofstream& out, + t_set* tset, + string elementName) { + t_field efield(tset->get_elem_type(), elementName); + generate_serialize_field(out, &efield, decontainerize(&efield, elementName)); +} + +/** + * Serializes the members of a list. + */ +void t_cocoa_generator::generate_serialize_list_element(ofstream& out, + t_list* tlist, + string index, + string listName) { + t_field efield(tlist->get_elem_type(), "[" + listName + " objectAtIndex: " + index + "]"); + generate_serialize_field(out, &efield, decontainerize(&efield, efield.get_name())); +} + + +/** + * Returns an Objective-C name + * + * @param ttype The type + * @param class_ref Do we want a Class reference istead of a type reference? + * @return Java type name, i.e. HashMap + */ +string t_cocoa_generator::type_name(t_type* ttype, bool class_ref) { + if (ttype->is_typedef()) { + return cocoa_prefix_ + ttype->get_name(); + } + + string result; + if (ttype->is_base_type()) { + return base_type_name((t_base_type*)ttype); + } else if (ttype->is_enum()) { + return "int"; + } else if (ttype->is_map()) { + result = "NSDictionary"; + } else if (ttype->is_set()) { + result = "NSSet"; + } else if (ttype->is_list()) { + result = "NSArray"; + } else { + // Check for prefix + t_program* program = ttype->get_program(); + if (program != NULL) { + result = program->get_namespace("cocoa") + ttype->get_name(); + } else { + result = ttype->get_name(); + } + } + + if (!class_ref) { + result += " *"; + } + return result; +} + +/** + * Returns the Objective-C type that corresponds to the thrift type. + * + * @param tbase The base type + */ +string t_cocoa_generator::base_type_name(t_base_type* type) { + t_base_type::t_base tbase = type->get_base(); + + switch (tbase) { + case t_base_type::TYPE_VOID: + return "void"; + case t_base_type::TYPE_STRING: + if (type->is_binary()) { + return "NSData *"; + } else { + return "NSString *"; + } + case t_base_type::TYPE_BOOL: + return "BOOL"; + case t_base_type::TYPE_BYTE: + return "uint8_t"; + case t_base_type::TYPE_I16: + return"int16_t"; + case t_base_type::TYPE_I32: + return "int32_t"; + case t_base_type::TYPE_I64: + return"int64_t"; + case t_base_type::TYPE_DOUBLE: + return "double"; + default: + throw "compiler error: no objective-c name for base type " + t_base_type::t_base_name(tbase); + } +} + + +/** + * Spit out code that evaluates to the specified constant value. + */ +string t_cocoa_generator::render_const_value(string name, + t_type* type, + t_const_value* value, + bool containerize_it) { + type = get_true_type(type); + std::ostringstream render; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + render << "@\"" << get_escaped_string(value) << '"'; + break; + case t_base_type::TYPE_BOOL: + render << ((value->get_integer() > 0) ? "YES" : "NO"); + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + render << value->get_integer(); + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() == t_const_value::CV_INTEGER) { + render << value->get_integer(); + } else { + render << value->get_double(); + } + break; + default: + throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + render << value->get_integer(); + } else if (type->is_struct() || type->is_xception()) { + render << "[[" << type_name(type, true) << " alloc] initWith"; + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + const map& val = value->get_map(); + map::const_iterator v_iter; + bool first = true; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + if (first) { + render << capitalize(v_iter->first->get_string()); + first = false; + } else { + render << " " << v_iter->first->get_string(); + } + render << ": " << render_const_value(name, field_type, v_iter->second); + } + render << "]"; + } else if (type->is_map()) { + render << "[[NSDictionary alloc] initWithObjectsAndKeys: "; + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + const map& val = value->get_map(); + map::const_iterator v_iter; + bool first = true; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + string key = render_const_value(name, ktype, v_iter->first, true); + string val = render_const_value(name, vtype, v_iter->second, true); + if (first) { + first = false; + } else { + render << ", "; + } + render << val << ", " << key; + } + render << ", nil]"; + } else if (type->is_list()) { + render << "[[NSArray alloc] initWithObjects: "; + t_type * etype = ((t_list*)type)->get_elem_type(); + const vector& val = value->get_list(); + bool first = true; + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + if (first) { + first = false; + } else { + render << ", "; + } + render << render_const_value(name, etype, *v_iter, true); + } + render << ", nil]"; + } else if (type->is_set()) { + render << "[[NSSet alloc] initWithObjects: "; + t_type * etype = ((t_set*)type)->get_elem_type(); + const vector& val = value->get_list(); + bool first = true; + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + if (first) { + first = false; + } else { + render << ", "; + } + render << render_const_value(name, etype, *v_iter, true); + } + render << ", nil]"; + } else { + throw "don't know how to render constant for type: " + type->get_name(); + } + + if (containerize_it) { + return containerize(type, render.str()); + } + + return render.str(); +} + + +/** + * Declares a field. + * + * @param ttype The type + */ +string t_cocoa_generator::declare_field(t_field* tfield) { + return type_name(tfield->get_type()) + " __" + tfield->get_name() + ";"; +} + +/** + * Renders a function signature + * + * @param tfunction Function definition + * @return String of rendered function definition + */ +string t_cocoa_generator::function_signature(t_function* tfunction) { + t_type* ttype = tfunction->get_returntype(); + std::string result = + "(" + type_name(ttype) + ") " + tfunction->get_name() + argument_list(tfunction->get_arglist()); + return result; +} + + +/** + * Renders a colon separated list of types and names, suitable for an + * objective-c parameter list + */ +string t_cocoa_generator::argument_list(t_struct* tstruct) { + string result = ""; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + result += " "; + } + result += ": (" + type_name((*f_iter)->get_type()) + ") " + (*f_iter)->get_name(); + } + return result; +} + + +/** + * Converts the parse type to an Objective-C enum string for the given type. + */ +string t_cocoa_generator::type_to_enum(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + return "TType_STRING"; + case t_base_type::TYPE_BOOL: + return "TType_BOOL"; + case t_base_type::TYPE_BYTE: + return "TType_BYTE"; + case t_base_type::TYPE_I16: + return "TType_I16"; + case t_base_type::TYPE_I32: + return "TType_I32"; + case t_base_type::TYPE_I64: + return "TType_I64"; + case t_base_type::TYPE_DOUBLE: + return "TType_DOUBLE"; + } + } else if (type->is_enum()) { + return "TType_I32"; + } else if (type->is_struct() || type->is_xception()) { + return "TType_STRUCT"; + } else if (type->is_map()) { + return "TType_MAP"; + } else if (type->is_set()) { + return "TType_SET"; + } else if (type->is_list()) { + return "TType_LIST"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + + +/** + * Returns a format string specifier for the supplied parse type. + */ +string t_cocoa_generator::format_string_for_type(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + return "\\\"%@\\\""; + case t_base_type::TYPE_BOOL: + return "%i"; + case t_base_type::TYPE_BYTE: + return "%i"; + case t_base_type::TYPE_I16: + return "%hi"; + case t_base_type::TYPE_I32: + return "%i"; + case t_base_type::TYPE_I64: + return "%qi"; + case t_base_type::TYPE_DOUBLE: + return "%f"; + } + } else if (type->is_enum()) { + return "%i"; + } else if (type->is_struct() || type->is_xception()) { + return "%@"; + } else if (type->is_map()) { + return "%@"; + } else if (type->is_set()) { + return "%@"; + } else if (type->is_list()) { + return "%@"; + } + + throw "INVALID TYPE IN format_string_for_type: " + type->get_name(); +} + +/** + * Generate a call to a field's setter. + * + * @param tfield Field the setter is being called on + * @param fieldName Name of variable to pass to setter + */ + +string t_cocoa_generator::call_field_setter(t_field* tfield, string fieldName) { + return "[self set" + capitalize(tfield->get_name()) + ": " + fieldName + "];"; +} + + +THRIFT_REGISTER_GENERATOR(cocoa, "Cocoa", ""); diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc new file mode 100644 index 00000000..67a4bd4e --- /dev/null +++ b/compiler/cpp/src/generate/t_cpp_generator.cc @@ -0,0 +1,3003 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#include + +#include "platform.h" +#include "t_oop_generator.h" +using namespace std; + + +/** + * C++ code generator. This is legitimacy incarnate. + * + */ +class t_cpp_generator : public t_oop_generator { + public: + t_cpp_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_oop_generator(program) + { + std::map::const_iterator iter; + + iter = parsed_options.find("dense"); + gen_dense_ = (iter != parsed_options.end()); + + iter = parsed_options.find("include_prefix"); + use_include_prefix_ = (iter != parsed_options.end()); + + out_dir_base_ = "gen-cpp"; + } + + /** + * Init and close methods + */ + + void init_generator(); + void close_generator(); + + void generate_consts(std::vector consts); + + /** + * Program-level generation functions + */ + + void generate_typedef(t_typedef* ttypedef); + void generate_enum(t_enum* tenum); + void generate_struct(t_struct* tstruct) { + generate_cpp_struct(tstruct, false); + } + void generate_xception(t_struct* txception) { + generate_cpp_struct(txception, true); + } + void generate_cpp_struct(t_struct* tstruct, bool is_exception); + + void generate_service(t_service* tservice); + + void print_const_value(std::ofstream& out, std::string name, t_type* type, t_const_value* value); + std::string render_const_value(std::ofstream& out, std::string name, t_type* type, t_const_value* value); + + void generate_struct_definition (std::ofstream& out, t_struct* tstruct, bool is_exception=false, bool pointers=false, bool read=true, bool write=true); + void generate_struct_fingerprint (std::ofstream& out, t_struct* tstruct, bool is_definition); + void generate_struct_reader (std::ofstream& out, t_struct* tstruct, bool pointers=false); + void generate_struct_writer (std::ofstream& out, t_struct* tstruct, bool pointers=false); + void generate_struct_result_writer (std::ofstream& out, t_struct* tstruct, bool pointers=false); + + /** + * Service-level generation functions + */ + + void generate_service_interface (t_service* tservice); + void generate_service_null (t_service* tservice); + void generate_service_multiface (t_service* tservice); + void generate_service_helpers (t_service* tservice); + void generate_service_client (t_service* tservice); + void generate_service_processor (t_service* tservice); + void generate_service_skeleton (t_service* tservice); + void generate_process_function (t_service* tservice, t_function* tfunction); + void generate_function_helpers (t_service* tservice, t_function* tfunction); + + /** + * Serialization constructs + */ + + void generate_deserialize_field (std::ofstream& out, + t_field* tfield, + std::string prefix="", + std::string suffix=""); + + void generate_deserialize_struct (std::ofstream& out, + t_struct* tstruct, + std::string prefix=""); + + void generate_deserialize_container (std::ofstream& out, + t_type* ttype, + std::string prefix=""); + + void generate_deserialize_set_element (std::ofstream& out, + t_set* tset, + std::string prefix=""); + + void generate_deserialize_map_element (std::ofstream& out, + t_map* tmap, + std::string prefix=""); + + void generate_deserialize_list_element (std::ofstream& out, + t_list* tlist, + std::string prefix, + bool push_back, + std::string index); + + void generate_serialize_field (std::ofstream& out, + t_field* tfield, + std::string prefix="", + std::string suffix=""); + + void generate_serialize_struct (std::ofstream& out, + t_struct* tstruct, + std::string prefix=""); + + void generate_serialize_container (std::ofstream& out, + t_type* ttype, + std::string prefix=""); + + void generate_serialize_map_element (std::ofstream& out, + t_map* tmap, + std::string iter); + + void generate_serialize_set_element (std::ofstream& out, + t_set* tmap, + std::string iter); + + void generate_serialize_list_element (std::ofstream& out, + t_list* tlist, + std::string iter); + + /** + * Helper rendering functions + */ + + std::string namespace_prefix(std::string ns); + std::string namespace_open(std::string ns); + std::string namespace_close(std::string ns); + std::string type_name(t_type* ttype, bool in_typedef=false, bool arg=false); + std::string base_type_name(t_base_type::t_base tbase); + std::string declare_field(t_field* tfield, bool init=false, bool pointer=false, bool constant=false, bool reference=false); + std::string function_signature(t_function* tfunction, std::string prefix="", bool name_params=true); + std::string argument_list(t_struct* tstruct, bool name_params=true); + std::string type_to_enum(t_type* ttype); + std::string local_reflection_name(const char*, t_type* ttype, bool external=false); + + // These handles checking gen_dense_ and checking for duplicates. + void generate_local_reflection(std::ofstream& out, t_type* ttype, bool is_definition); + void generate_local_reflection_pointer(std::ofstream& out, t_type* ttype); + + bool is_complex_type(t_type* ttype) { + ttype = get_true_type(ttype); + + return + ttype->is_container() || + ttype->is_struct() || + ttype->is_xception() || + (ttype->is_base_type() && (((t_base_type*)ttype)->get_base() == t_base_type::TYPE_STRING)); + } + + void set_use_include_prefix(bool use_include_prefix) { + use_include_prefix_ = use_include_prefix; + } + + private: + /** + * Returns the include prefix to use for a file generated by program, or the + * empty string if no include prefix should be used. + */ + std::string get_include_prefix(const t_program& program) const; + + /** + * True iff we should generate local reflection metadata for TDenseProtocol. + */ + bool gen_dense_; + + /** + * True iff we should use a path prefix in our #include statements for other + * thrift-generated header files. + */ + bool use_include_prefix_; + + /** + * Strings for namespace, computed once up front then used directly + */ + + std::string ns_open_; + std::string ns_close_; + + /** + * File streams, stored here to avoid passing them as parameters to every + * function. + */ + + std::ofstream f_types_; + std::ofstream f_types_impl_; + std::ofstream f_header_; + std::ofstream f_service_; + + /** + * When generating local reflections, make sure we don't generate duplicates. + */ + std::set reflected_fingerprints_; +}; + + +/** + * Prepares for file generation by opening up the necessary file output + * streams. + * + * @param tprogram The program to generate + */ +void t_cpp_generator::init_generator() { + // Make output directory + MKDIR(get_out_dir().c_str()); + + // Make output file + string f_types_name = get_out_dir()+program_name_+"_types.h"; + f_types_.open(f_types_name.c_str()); + + string f_types_impl_name = get_out_dir()+program_name_+"_types.cpp"; + f_types_impl_.open(f_types_impl_name.c_str()); + + // Print header + f_types_ << + autogen_comment(); + f_types_impl_ << + autogen_comment(); + + // Start ifndef + f_types_ << + "#ifndef " << program_name_ << "_TYPES_H" << endl << + "#define " << program_name_ << "_TYPES_H" << endl << + endl; + + // Include base types + f_types_ << + "#include " << endl << + "#include " << endl << + "#include " << endl << + endl; + + // Include other Thrift includes + const vector& includes = program_->get_includes(); + for (size_t i = 0; i < includes.size(); ++i) { + f_types_ << + "#include \"" << get_include_prefix(*(includes[i])) << + includes[i]->get_name() << "_types.h\"" << endl; + } + f_types_ << endl; + + // Include custom headers + const vector& cpp_includes = program_->get_cpp_includes(); + for (size_t i = 0; i < cpp_includes.size(); ++i) { + if (cpp_includes[i][0] == '<') { + f_types_ << + "#include " << cpp_includes[i] << endl; + } else { + f_types_ << + "#include \"" << cpp_includes[i] << "\"" << endl; + } + } + f_types_ << + endl; + + // Include the types file + f_types_impl_ << + "#include \"" << get_include_prefix(*get_program()) << program_name_ << + "_types.h\"" << endl << + endl; + + // If we are generating local reflection metadata, we need to include + // the definition of TypeSpec. + if (gen_dense_) { + f_types_impl_ << + "#include " << endl << + endl; + } + + // Open namespace + ns_open_ = namespace_open(program_->get_namespace("cpp")); + ns_close_ = namespace_close(program_->get_namespace("cpp")); + + f_types_ << + ns_open_ << endl << + endl; + + f_types_impl_ << + ns_open_ << endl << + endl; +} + +/** + * Closes the output files. + */ +void t_cpp_generator::close_generator() { + // Close namespace + f_types_ << + ns_close_ << endl << + endl; + f_types_impl_ << + ns_close_ << endl; + + // Close ifndef + f_types_ << + "#endif" << endl; + + // Close output file + f_types_.close(); + f_types_impl_.close(); +} + +/** + * Generates a typedef. This is just a simple 1-liner in C++ + * + * @param ttypedef The type definition + */ +void t_cpp_generator::generate_typedef(t_typedef* ttypedef) { + f_types_ << + indent() << "typedef " << type_name(ttypedef->get_type(), true) << " " << ttypedef->get_symbolic() << ";" << endl << + endl; +} + +/** + * Generates code for an enumerated type. In C++, this is essentially the same + * as the thrift definition itself, using the enum keyword in C++. + * + * @param tenum The enumeration + */ +void t_cpp_generator::generate_enum(t_enum* tenum) { + f_types_ << + indent() << "enum " << tenum->get_name() << " {" << endl; + indent_up(); + + vector constants = tenum->get_constants(); + vector::iterator c_iter; + bool first = true; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if (first) { + first = false; + } else { + f_types_ << + "," << endl; + } + f_types_ << + indent() << (*c_iter)->get_name(); + if ((*c_iter)->has_value()) { + f_types_ << + " = " << (*c_iter)->get_value(); + } + } + + indent_down(); + f_types_ << + endl << + "};" << endl << + endl; + + generate_local_reflection(f_types_, tenum, false); + generate_local_reflection(f_types_impl_, tenum, true); +} + +/** + * Generates a class that holds all the constants. + */ +void t_cpp_generator::generate_consts(std::vector consts) { + string f_consts_name = get_out_dir()+program_name_+"_constants.h"; + ofstream f_consts; + f_consts.open(f_consts_name.c_str()); + + string f_consts_impl_name = get_out_dir()+program_name_+"_constants.cpp"; + ofstream f_consts_impl; + f_consts_impl.open(f_consts_impl_name.c_str()); + + // Print header + f_consts << + autogen_comment(); + f_consts_impl << + autogen_comment(); + + // Start ifndef + f_consts << + "#ifndef " << program_name_ << "_CONSTANTS_H" << endl << + "#define " << program_name_ << "_CONSTANTS_H" << endl << + endl << + "#include \"" << get_include_prefix(*get_program()) << program_name_ << + "_types.h\"" << endl << + endl << + ns_open_ << endl << + endl; + + f_consts_impl << + "#include \"" << get_include_prefix(*get_program()) << program_name_ << + "_constants.h\"" << endl << + endl << + ns_open_ << endl << + endl; + + f_consts << + "class " << program_name_ << "Constants {" << endl << + " public:" << endl << + " " << program_name_ << "Constants();" << endl << + endl; + indent_up(); + vector::iterator c_iter; + for (c_iter = consts.begin(); c_iter != consts.end(); ++c_iter) { + string name = (*c_iter)->get_name(); + t_type* type = (*c_iter)->get_type(); + f_consts << + indent() << type_name(type) << " " << name << ";" << endl; + } + indent_down(); + f_consts << + "};" << endl; + + f_consts_impl << + "const " << program_name_ << "Constants g_" << program_name_ << "_constants;" << endl << + endl << + program_name_ << "Constants::" << program_name_ << "Constants() {" << endl; + indent_up(); + for (c_iter = consts.begin(); c_iter != consts.end(); ++c_iter) { + print_const_value(f_consts_impl, + (*c_iter)->get_name(), + (*c_iter)->get_type(), + (*c_iter)->get_value()); + } + indent_down(); + indent(f_consts_impl) << + "}" << endl; + + f_consts << + endl << + "extern const " << program_name_ << "Constants g_" << program_name_ << "_constants;" << endl << + endl << + ns_close_ << endl << + endl << + "#endif" << endl; + f_consts.close(); + + f_consts_impl << + endl << + ns_close_ << endl << + endl; +} + +/** + * Prints the value of a constant with the given type. Note that type checking + * is NOT performed in this function as it is always run beforehand using the + * validate_types method in main.cc + */ +void t_cpp_generator::print_const_value(ofstream& out, string name, t_type* type, t_const_value* value) { + type = get_true_type(type); + if (type->is_base_type()) { + string v2 = render_const_value(out, name, type, value); + indent(out) << name << " = " << v2 << ";" << endl << + endl; + } else if (type->is_enum()) { + indent(out) << name << " = (" << type_name(type) << ")" << value->get_integer() << ";" << endl << + endl; + } else if (type->is_struct() || type->is_xception()) { + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + string val = render_const_value(out, name, field_type, v_iter->second); + indent(out) << name << "." << v_iter->first->get_string() << " = " << val << ";" << endl; + indent(out) << name << ".__isset." << v_iter->first->get_string() << " = true;" << endl; + } + out << endl; + } else if (type->is_map()) { + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + string key = render_const_value(out, name, ktype, v_iter->first); + string val = render_const_value(out, name, vtype, v_iter->second); + indent(out) << name << ".insert(std::make_pair(" << key << ", " << val << "));" << endl; + } + out << endl; + } else if (type->is_list()) { + t_type* etype = ((t_list*)type)->get_elem_type(); + const vector& val = value->get_list(); + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + string val = render_const_value(out, name, etype, *v_iter); + indent(out) << name << ".push_back(" << val << ");" << endl; + } + out << endl; + } else if (type->is_set()) { + t_type* etype = ((t_set*)type)->get_elem_type(); + const vector& val = value->get_list(); + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + string val = render_const_value(out, name, etype, *v_iter); + indent(out) << name << ".insert(" << val << ");" << endl; + } + out << endl; + } else { + throw "INVALID TYPE IN print_const_value: " + type->get_name(); + } +} + +/** + * + */ +string t_cpp_generator::render_const_value(ofstream& out, string name, t_type* type, t_const_value* value) { + std::ostringstream render; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + render << '"' << get_escaped_string(value) << '"'; + break; + case t_base_type::TYPE_BOOL: + render << ((value->get_integer() > 0) ? "true" : "false"); + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + render << value->get_integer(); + break; + case t_base_type::TYPE_I64: + render << value->get_integer() << "LL"; + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() == t_const_value::CV_INTEGER) { + render << value->get_integer(); + } else { + render << value->get_double(); + } + break; + default: + throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + render << "(" << type_name(type) << ")" << value->get_integer(); + } else { + string t = tmp("tmp"); + indent(out) << type_name(type) << " " << t << ";" << endl; + print_const_value(out, t, type, value); + render << t; + } + + return render.str(); +} + +/** + * Generates a struct definition for a thrift data type. This is a class + * with data members and a read/write() function, plus a mirroring isset + * inner class. + * + * @param tstruct The struct definition + */ +void t_cpp_generator::generate_cpp_struct(t_struct* tstruct, bool is_exception) { + generate_struct_definition(f_types_, tstruct, is_exception); + generate_struct_fingerprint(f_types_impl_, tstruct, true); + generate_local_reflection(f_types_, tstruct, false); + generate_local_reflection(f_types_impl_, tstruct, true); + generate_local_reflection_pointer(f_types_impl_, tstruct); + generate_struct_reader(f_types_impl_, tstruct); + generate_struct_writer(f_types_impl_, tstruct); +} + +/** + * Writes the struct definition into the header file + * + * @param out Output stream + * @param tstruct The struct + */ +void t_cpp_generator::generate_struct_definition(ofstream& out, + t_struct* tstruct, + bool is_exception, + bool pointers, + bool read, + bool write) { + string extends = ""; + if (is_exception) { + extends = " : public apache::thrift::TException"; + } + + // Open struct def + out << + indent() << "class " << tstruct->get_name() << extends << " {" << endl << + indent() << " public:" << endl << + endl; + indent_up(); + + // Put the fingerprint up top for all to see. + generate_struct_fingerprint(out, tstruct, false); + + // Get members + vector::const_iterator m_iter; + const vector& members = tstruct->get_members(); + + if (!pointers) { + // Default constructor + indent(out) << + tstruct->get_name() << "()"; + + bool init_ctor = false; + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + t_type* t = get_true_type((*m_iter)->get_type()); + if (t->is_base_type()) { + string dval; + if (t->is_enum()) { + dval += "(" + type_name(t) + ")"; + } + dval += t->is_string() ? "\"\"" : "0"; + t_const_value* cv = (*m_iter)->get_value(); + if (cv != NULL) { + dval = render_const_value(out, (*m_iter)->get_name(), t, cv); + } + if (!init_ctor) { + init_ctor = true; + out << " : "; + out << (*m_iter)->get_name() << "(" << dval << ")"; + } else { + out << ", " << (*m_iter)->get_name() << "(" << dval << ")"; + } + } + } + out << " {" << endl; + indent_up(); + // TODO(dreiss): When everything else in Thrift is perfect, + // do more of these in the initializer list. + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + t_type* t = get_true_type((*m_iter)->get_type()); + + if (!t->is_base_type()) { + t_const_value* cv = (*m_iter)->get_value(); + if (cv != NULL) { + print_const_value(out, (*m_iter)->get_name(), t, cv); + } + } + } + scope_down(out); + } + + if (tstruct->annotations_.find("final") == tstruct->annotations_.end()) { + out << + endl << + indent() << "virtual ~" << tstruct->get_name() << "() throw() {}" << endl << endl; + } + + // Pointer to this structure's reflection local typespec. + if (gen_dense_) { + indent(out) << + "static apache::thrift::reflection::local::TypeSpec* local_reflection;" << + endl << endl; + } + + // Declare all fields + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + indent(out) << + declare_field(*m_iter, false, pointers && !(*m_iter)->get_type()->is_xception(), !read) << endl; + } + + // Isset struct has boolean fields, but only for non-required fields. + bool has_nonrequired_fields = false; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + if ((*m_iter)->get_req() != t_field::T_REQUIRED) + has_nonrequired_fields = true; + } + + if (has_nonrequired_fields && (!pointers || read)) { + out << + endl << + indent() << "struct __isset {" << endl; + indent_up(); + + indent(out) << + "__isset() : "; + bool first = true; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + if ((*m_iter)->get_req() == t_field::T_REQUIRED) { + continue; + } + if (first) { + first = false; + out << + (*m_iter)->get_name() << "(false)"; + } else { + out << + ", " << (*m_iter)->get_name() << "(false)"; + } + } + out << " {}" << endl; + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + if ((*m_iter)->get_req() != t_field::T_REQUIRED) { + indent(out) << + "bool " << (*m_iter)->get_name() << ";" << endl; + } + } + + indent_down(); + indent(out) << + "} __isset;" << endl; + } + + out << endl; + + if (!pointers) { + // Generate an equality testing operator. Make it inline since the compiler + // will do a better job than we would when deciding whether to inline it. + out << + indent() << "bool operator == (const " << tstruct->get_name() << " & " << + (members.size() > 0 ? "rhs" : "/* rhs */") << ") const" << endl; + scope_up(out); + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + // Most existing Thrift code does not use isset or optional/required, + // so we treat "default" fields as required. + if ((*m_iter)->get_req() != t_field::T_OPTIONAL) { + out << + indent() << "if (!(" << (*m_iter)->get_name() + << " == rhs." << (*m_iter)->get_name() << "))" << endl << + indent() << " return false;" << endl; + } else { + out << + indent() << "if (__isset." << (*m_iter)->get_name() + << " != rhs.__isset." << (*m_iter)->get_name() << ")" << endl << + indent() << " return false;" << endl << + indent() << "else if (__isset." << (*m_iter)->get_name() << " && !(" + << (*m_iter)->get_name() << " == rhs." << (*m_iter)->get_name() + << "))" << endl << + indent() << " return false;" << endl; + } + } + indent(out) << "return true;" << endl; + scope_down(out); + out << + indent() << "bool operator != (const " << tstruct->get_name() << " &rhs) const {" << endl << + indent() << " return !(*this == rhs);" << endl << + indent() << "}" << endl << endl; + + // Generate the declaration of a less-than operator. This must be + // implemented by the application developer if they wish to use it. (They + // will get a link error if they try to use it without an implementation.) + out << + indent() << "bool operator < (const " + << tstruct->get_name() << " & ) const;" << endl << endl; + } + if (read) { + out << + indent() << "uint32_t read(apache::thrift::protocol::TProtocol* iprot);" << endl; + } + if (write) { + out << + indent() << "uint32_t write(apache::thrift::protocol::TProtocol* oprot) const;" << endl; + } + out << endl; + + indent_down(); + indent(out) << + "};" << endl << + endl; +} + +/** + * Writes the fingerprint of a struct to either the header or implementation. + * + * @param out Output stream + * @param tstruct The struct + */ +void t_cpp_generator::generate_struct_fingerprint(ofstream& out, + t_struct* tstruct, + bool is_definition) { + string stat, nspace, comment; + if (is_definition) { + stat = ""; + nspace = tstruct->get_name() + "::"; + comment = " "; + } else { + stat = "static "; + nspace = ""; + comment = "; // "; + } + + if (tstruct->has_fingerprint()) { + out << + indent() << stat << "const char* " << nspace + << "ascii_fingerprint" << comment << "= \"" << + tstruct->get_ascii_fingerprint() << "\";" << endl << + indent() << stat << "const uint8_t " << nspace << + "binary_fingerprint[" << t_type::fingerprint_len << "]" << comment << "= {"; + const char* comma = ""; + for (int i = 0; i < t_type::fingerprint_len; i++) { + out << comma << "0x" << t_struct::byte_to_hex(tstruct->get_binary_fingerprint()[i]); + comma = ","; + } + out << "};" << endl << endl; + } +} + +/** + * Writes the local reflection of a type (either declaration or definition). + */ +void t_cpp_generator::generate_local_reflection(std::ofstream& out, + t_type* ttype, + bool is_definition) { + if (!gen_dense_) { + return; + } + ttype = get_true_type(ttype); + assert(ttype->has_fingerprint()); + string key = ttype->get_ascii_fingerprint() + (is_definition ? "-defn" : "-decl"); + // Note that we have generated this fingerprint. If we already did, bail out. + if (!reflected_fingerprints_.insert(key).second) { + return; + } + // Let each program handle its own structures. + if (ttype->get_program() != NULL && ttype->get_program() != program_) { + return; + } + + // Do dependencies. + if (ttype->is_list()) { + generate_local_reflection(out, ((t_list*)ttype)->get_elem_type(), is_definition); + } else if (ttype->is_set()) { + generate_local_reflection(out, ((t_set*)ttype)->get_elem_type(), is_definition); + } else if (ttype->is_map()) { + generate_local_reflection(out, ((t_map*)ttype)->get_key_type(), is_definition); + generate_local_reflection(out, ((t_map*)ttype)->get_val_type(), is_definition); + } else if (ttype->is_struct() || ttype->is_xception()) { + // Hacky hacky. For efficiency and convenience, we need a dummy "T_STOP" + // type at the end of our typespec array. Unfortunately, there is no + // T_STOP type, so we use the global void type, and special case it when + // generating its typespec. + + const vector& members = ((t_struct*)ttype)->get_sorted_members(); + vector::const_iterator m_iter; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + generate_local_reflection(out, (**m_iter).get_type(), is_definition); + } + generate_local_reflection(out, g_type_void, is_definition); + + // For definitions of structures, do the arrays of metas and field specs also. + if (is_definition) { + out << + indent() << "apache::thrift::reflection::local::FieldMeta" << endl << + indent() << local_reflection_name("metas", ttype) <<"[] = {" << endl; + indent_up(); + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + indent(out) << "{ " << (*m_iter)->get_key() << ", " << + (((*m_iter)->get_req() == t_field::T_OPTIONAL) ? "true" : "false") << + " }," << endl; + } + // Zero for the T_STOP marker. + indent(out) << "{ 0, false }" << endl << "};" << endl; + indent_down(); + + out << + indent() << "apache::thrift::reflection::local::TypeSpec*" << endl << + indent() << local_reflection_name("specs", ttype) <<"[] = {" << endl; + indent_up(); + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + indent(out) << "&" << + local_reflection_name("typespec", (*m_iter)->get_type(), true) << "," << endl; + } + indent(out) << "&" << + local_reflection_name("typespec", g_type_void) << "," << endl; + indent_down(); + indent(out) << "};" << endl; + } + } + + out << + indent() << "// " << ttype->get_fingerprint_material() << endl << + indent() << (is_definition ? "" : "extern ") << + "apache::thrift::reflection::local::TypeSpec" << endl << + local_reflection_name("typespec", ttype) << + (is_definition ? "(" : ";") << endl; + + if (!is_definition) { + out << endl; + return; + } + + indent_up(); + + if (ttype->is_void()) { + indent(out) << "apache::thrift::protocol::T_STOP"; + } else { + indent(out) << type_to_enum(ttype); + } + + if (ttype->is_struct()) { + out << "," << endl << + indent() << type_name(ttype) << "::binary_fingerprint," << endl << + indent() << local_reflection_name("metas", ttype) << "," << endl << + indent() << local_reflection_name("specs", ttype); + } else if (ttype->is_list()) { + out << "," << endl << + indent() << "&" << local_reflection_name("typespec", ((t_list*)ttype)->get_elem_type()) << "," << endl << + indent() << "NULL"; + } else if (ttype->is_set()) { + out << "," << endl << + indent() << "&" << local_reflection_name("typespec", ((t_set*)ttype)->get_elem_type()) << "," << endl << + indent() << "NULL"; + } else if (ttype->is_map()) { + out << "," << endl << + indent() << "&" << local_reflection_name("typespec", ((t_map*)ttype)->get_key_type()) << "," << endl << + indent() << "&" << local_reflection_name("typespec", ((t_map*)ttype)->get_val_type()); + } + + out << ");" << endl << endl; + + indent_down(); +} + +/** + * Writes the structure's static pointer to its local reflection typespec + * into the implementation file. + */ +void t_cpp_generator::generate_local_reflection_pointer(std::ofstream& out, + t_type* ttype) { + if (!gen_dense_) { + return; + } + indent(out) << + "apache::thrift::reflection::local::TypeSpec* " << + ttype->get_name() << "::local_reflection = " << endl << + indent() << " &" << local_reflection_name("typespec", ttype) << ";" << + endl << endl; +} + +/** + * Makes a helper function to gen a struct reader. + * + * @param out Stream to write to + * @param tstruct The struct + */ +void t_cpp_generator::generate_struct_reader(ofstream& out, + t_struct* tstruct, + bool pointers) { + indent(out) << + "uint32_t " << tstruct->get_name() << "::read(apache::thrift::protocol::TProtocol* iprot) {" << endl; + indent_up(); + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + // Declare stack tmp variables + out << + endl << + indent() << "uint32_t xfer = 0;" << endl << + indent() << "std::string fname;" << endl << + indent() << "apache::thrift::protocol::TType ftype;" << endl << + indent() << "int16_t fid;" << endl << + endl << + indent() << "xfer += iprot->readStructBegin(fname);" << endl << + endl << + indent() << "using apache::thrift::protocol::TProtocolException;" << endl << + endl; + + // Required variables aren't in __isset, so we need tmp vars to check them. + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_req() == t_field::T_REQUIRED) + indent(out) << "bool isset_" << (*f_iter)->get_name() << " = false;" << endl; + } + out << endl; + + + // Loop over reading in fields + indent(out) << + "while (true)" << endl; + scope_up(out); + + // Read beginning field marker + indent(out) << + "xfer += iprot->readFieldBegin(fname, ftype, fid);" << endl; + + // Check for field STOP marker + out << + indent() << "if (ftype == apache::thrift::protocol::T_STOP) {" << endl << + indent() << " break;" << endl << + indent() << "}" << endl; + + // Switch statement on the field we are reading + indent(out) << + "switch (fid)" << endl; + + scope_up(out); + + // Generate deserialization code for known cases + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + indent(out) << + "case " << (*f_iter)->get_key() << ":" << endl; + indent_up(); + indent(out) << + "if (ftype == " << type_to_enum((*f_iter)->get_type()) << ") {" << endl; + indent_up(); + + const char *isset_prefix = + ((*f_iter)->get_req() != t_field::T_REQUIRED) ? "this->__isset." : "isset_"; + +#if 0 + // This code throws an exception if the same field is encountered twice. + // We've decided to leave it out for performance reasons. + // TODO(dreiss): Generate this code and "if" it out to make it easier + // for people recompiling thrift to include it. + out << + indent() << "if (" << isset_prefix << (*f_iter)->get_name() << ")" << endl << + indent() << " throw TProtocolException(TProtocolException::INVALID_DATA);" << endl; +#endif + + if (pointers && !(*f_iter)->get_type()->is_xception()) { + generate_deserialize_field(out, *f_iter, "(*(this->", "))"); + } else { + generate_deserialize_field(out, *f_iter, "this->"); + } + out << + indent() << isset_prefix << (*f_iter)->get_name() << " = true;" << endl; + indent_down(); + out << + indent() << "} else {" << endl << + indent() << " xfer += iprot->skip(ftype);" << endl << + // TODO(dreiss): Make this an option when thrift structs + // have a common base class. + // indent() << " throw TProtocolException(TProtocolException::INVALID_DATA);" << endl << + indent() << "}" << endl << + indent() << "break;" << endl; + indent_down(); + } + + // In the default case we skip the field + out << + indent() << "default:" << endl << + indent() << " xfer += iprot->skip(ftype);" << endl << + indent() << " break;" << endl; + + scope_down(out); + + // Read field end marker + indent(out) << + "xfer += iprot->readFieldEnd();" << endl; + + scope_down(out); + + out << + endl << + indent() << "xfer += iprot->readStructEnd();" << endl; + + // Throw if any required fields are missing. + // We do this after reading the struct end so that + // there might possibly be a chance of continuing. + out << endl; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_req() == t_field::T_REQUIRED) + out << + indent() << "if (!isset_" << (*f_iter)->get_name() << ')' << endl << + indent() << " throw TProtocolException(TProtocolException::INVALID_DATA);" << endl; + } + + indent(out) << "return xfer;" << endl; + + indent_down(); + indent(out) << + "}" << endl << endl; +} + +/** + * Generates the write function. + * + * @param out Stream to write to + * @param tstruct The struct + */ +void t_cpp_generator::generate_struct_writer(ofstream& out, + t_struct* tstruct, + bool pointers) { + string name = tstruct->get_name(); + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + indent(out) << + "uint32_t " << tstruct->get_name() << "::write(apache::thrift::protocol::TProtocol* oprot) const {" << endl; + indent_up(); + + out << + indent() << "uint32_t xfer = 0;" << endl; + + indent(out) << + "xfer += oprot->writeStructBegin(\"" << name << "\");" << endl; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_req() == t_field::T_OPTIONAL) { + indent(out) << "if (this->__isset." << (*f_iter)->get_name() << ") {" << endl; + indent_up(); + } + // Write field header + out << + indent() << "xfer += oprot->writeFieldBegin(" << + "\"" << (*f_iter)->get_name() << "\", " << + type_to_enum((*f_iter)->get_type()) << ", " << + (*f_iter)->get_key() << ");" << endl; + // Write field contents + if (pointers) { + generate_serialize_field(out, *f_iter, "(*(this->", "))"); + } else { + generate_serialize_field(out, *f_iter, "this->"); + } + // Write field closer + indent(out) << + "xfer += oprot->writeFieldEnd();" << endl; + if ((*f_iter)->get_req() == t_field::T_OPTIONAL) { + indent_down(); + indent(out) << '}' << endl; + } + } + + // Write the struct map + out << + indent() << "xfer += oprot->writeFieldStop();" << endl << + indent() << "xfer += oprot->writeStructEnd();" << endl << + indent() << "return xfer;" << endl; + + indent_down(); + indent(out) << + "}" << endl << + endl; +} + +/** + * Struct writer for result of a function, which can have only one of its + * fields set and does a conditional if else look up into the __isset field + * of the struct. + * + * @param out Output stream + * @param tstruct The result struct + */ +void t_cpp_generator::generate_struct_result_writer(ofstream& out, + t_struct* tstruct, + bool pointers) { + string name = tstruct->get_name(); + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + indent(out) << + "uint32_t " << tstruct->get_name() << "::write(apache::thrift::protocol::TProtocol* oprot) const {" << endl; + indent_up(); + + out << + endl << + indent() << "uint32_t xfer = 0;" << endl << + endl; + + indent(out) << + "xfer += oprot->writeStructBegin(\"" << name << "\");" << endl; + + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + out << + endl << + indent() << "if "; + } else { + out << + " else if "; + } + + out << "(this->__isset." << (*f_iter)->get_name() << ") {" << endl; + + indent_up(); + + // Write field header + out << + indent() << "xfer += oprot->writeFieldBegin(" << + "\"" << (*f_iter)->get_name() << "\", " << + type_to_enum((*f_iter)->get_type()) << ", " << + (*f_iter)->get_key() << ");" << endl; + // Write field contents + if (pointers) { + generate_serialize_field(out, *f_iter, "(*(this->", "))"); + } else { + generate_serialize_field(out, *f_iter, "this->"); + } + // Write field closer + indent(out) << "xfer += oprot->writeFieldEnd();" << endl; + + indent_down(); + indent(out) << "}"; + } + + // Write the struct map + out << + endl << + indent() << "xfer += oprot->writeFieldStop();" << endl << + indent() << "xfer += oprot->writeStructEnd();" << endl << + indent() << "return xfer;" << endl; + + indent_down(); + indent(out) << + "}" << endl << + endl; +} + +/** + * Generates a thrift service. In C++, this comprises an entirely separate + * header and source file. The header file defines the methods and includes + * the data types defined in the main header file, and the implementation + * file contains implementations of the basic printer and default interfaces. + * + * @param tservice The service definition + */ +void t_cpp_generator::generate_service(t_service* tservice) { + string svcname = tservice->get_name(); + + // Make output files + string f_header_name = get_out_dir()+svcname+".h"; + f_header_.open(f_header_name.c_str()); + + // Print header file includes + f_header_ << + autogen_comment(); + f_header_ << + "#ifndef " << svcname << "_H" << endl << + "#define " << svcname << "_H" << endl << + endl << + "#include " << endl << + "#include \"" << get_include_prefix(*get_program()) << program_name_ << + "_types.h\"" << endl; + + t_service* extends_service = tservice->get_extends(); + if (extends_service != NULL) { + f_header_ << + "#include \"" << get_include_prefix(*(extends_service->get_program())) << + extends_service->get_name() << ".h\"" << endl; + } + + f_header_ << + endl << + ns_open_ << endl << + endl; + + // Service implementation file includes + string f_service_name = get_out_dir()+svcname+".cpp"; + f_service_.open(f_service_name.c_str()); + f_service_ << + autogen_comment(); + f_service_ << + "#include \"" << get_include_prefix(*get_program()) << svcname << ".h\"" << + endl << + endl << + ns_open_ << endl << + endl; + + // Generate all the components + generate_service_interface(tservice); + generate_service_null(tservice); + generate_service_helpers(tservice); + generate_service_client(tservice); + generate_service_processor(tservice); + generate_service_multiface(tservice); + generate_service_skeleton(tservice); + + // Close the namespace + f_service_ << + ns_close_ << endl << + endl; + f_header_ << + ns_close_ << endl << + endl; + f_header_ << + "#endif" << endl; + + // Close the files + f_service_.close(); + f_header_.close(); +} + +/** + * Generates helper functions for a service. Basically, this generates types + * for all the arguments and results to functions. + * + * @param tservice The service to generate a header definition for + */ +void t_cpp_generator::generate_service_helpers(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* ts = (*f_iter)->get_arglist(); + string name_orig = ts->get_name(); + + ts->set_name(tservice->get_name() + "_" + (*f_iter)->get_name() + "_args"); + generate_struct_definition(f_header_, ts, false); + generate_struct_reader(f_service_, ts); + generate_struct_writer(f_service_, ts); + ts->set_name(tservice->get_name() + "_" + (*f_iter)->get_name() + "_pargs"); + generate_struct_definition(f_header_, ts, false, true, false, true); + generate_struct_writer(f_service_, ts, true); + ts->set_name(name_orig); + + generate_function_helpers(tservice, *f_iter); + } +} + +/** + * Generates a service interface definition. + * + * @param tservice The service to generate a header definition for + */ +void t_cpp_generator::generate_service_interface(t_service* tservice) { + string extends = ""; + if (tservice->get_extends() != NULL) { + extends = " : virtual public " + type_name(tservice->get_extends()) + "If"; + } + f_header_ << + "class " << service_name_ << "If" << extends << " {" << endl << + " public:" << endl; + indent_up(); + f_header_ << + indent() << "virtual ~" << service_name_ << "If() {}" << endl; + + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_header_ << + indent() << "virtual " << function_signature(*f_iter) << " = 0;" << endl; + } + indent_down(); + f_header_ << + "};" << endl << endl; +} + +/** + * Generates a null implementation of the service. + * + * @param tservice The service to generate a header definition for + */ +void t_cpp_generator::generate_service_null(t_service* tservice) { + string extends = ""; + if (tservice->get_extends() != NULL) { + extends = " , virtual public " + type_name(tservice->get_extends()) + "Null"; + } + f_header_ << + "class " << service_name_ << "Null : virtual public " << service_name_ << "If" << extends << " {" << endl << + " public:" << endl; + indent_up(); + f_header_ << + indent() << "virtual ~" << service_name_ << "Null() {}" << endl; + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_header_ << + indent() << function_signature(*f_iter, "", false) << " {" << endl; + indent_up(); + t_type* returntype = (*f_iter)->get_returntype(); + if (returntype->is_void()) { + f_header_ << + indent() << "return;" << endl; + } else if (is_complex_type(returntype)) { + f_header_ << + indent() << "return;" << endl; + } else { + t_field returnfield(returntype, "_return"); + f_header_ << + indent() << declare_field(&returnfield, true) << endl << + indent() << "return _return;" << endl; + } + indent_down(); + f_header_ << + indent() << "}" << endl; + } + indent_down(); + f_header_ << + "};" << endl << endl; +} + + +/** + * Generates a multiface, which is a single server that just takes a set + * of objects implementing the interface and calls them all, returning the + * value of the last one to be called. + * + * @param tservice The service to generate a multiserver for. + */ +void t_cpp_generator::generate_service_multiface(t_service* tservice) { + // Generate the dispatch methods + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + string extends = ""; + string extends_multiface = ""; + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + extends_multiface = ", public " + extends + "Multiface"; + } + + string list_type = string("std::vector >"; + + // Generate the header portion + f_header_ << + "class " << service_name_ << "Multiface : " << + "virtual public " << service_name_ << "If" << + extends_multiface << " {" << endl << + " public:" << endl; + indent_up(); + f_header_ << + indent() << service_name_ << "Multiface(" << list_type << "& ifaces) : ifaces_(ifaces) {" << endl; + if (!extends.empty()) { + f_header_ << + indent() << " std::vector >::iterator iter;" << endl << + indent() << " for (iter = ifaces.begin(); iter != ifaces.end(); ++iter) {" << endl << + indent() << " " << extends << "Multiface::add(*iter);" << endl << + indent() << " }" << endl; + } + f_header_ << + indent() << "}" << endl << + indent() << "virtual ~" << service_name_ << "Multiface() {}" << endl; + indent_down(); + + // Protected data members + f_header_ << + " protected:" << endl; + indent_up(); + f_header_ << + indent() << list_type << " ifaces_;" << endl << + indent() << service_name_ << "Multiface() {}" << endl << + indent() << "void add(boost::shared_ptr<" << service_name_ << "If> iface) {" << endl; + if (!extends.empty()) { + f_header_ << + indent() << " " << extends << "Multiface::add(iface);" << endl; + } + f_header_ << + indent() << " ifaces_.push_back(iface);" << endl << + indent() << "}" << endl; + indent_down(); + + f_header_ << + indent() << " public:" << endl; + indent_up(); + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* arglist = (*f_iter)->get_arglist(); + const vector& args = arglist->get_members(); + vector::const_iterator a_iter; + + string call = string("ifaces_[i]->") + (*f_iter)->get_name() + "("; + bool first = true; + if (is_complex_type((*f_iter)->get_returntype())) { + call += "_return"; + first = false; + } + for (a_iter = args.begin(); a_iter != args.end(); ++a_iter) { + if (first) { + first = false; + } else { + call += ", "; + } + call += (*a_iter)->get_name(); + } + call += ")"; + + f_header_ << + indent() << function_signature(*f_iter) << " {" << endl; + indent_up(); + f_header_ << + indent() << "uint32_t sz = ifaces_.size();" << endl << + indent() << "for (uint32_t i = 0; i < sz; ++i) {" << endl; + if (!(*f_iter)->get_returntype()->is_void()) { + f_header_ << + indent() << " if (i == sz - 1) {" << endl; + if (is_complex_type((*f_iter)->get_returntype())) { + f_header_ << + indent() << " " << call << ";" << endl << + indent() << " return;" << endl; + } else { + f_header_ << + indent() << " return " << call << ";" << endl; + } + f_header_ << + indent() << " } else {" << endl << + indent() << " " << call << ";" << endl << + indent() << " }" << endl; + } else { + f_header_ << + indent() << " " << call << ";" << endl; + } + + f_header_ << + indent() << "}" << endl; + + indent_down(); + f_header_ << + indent() << "}" << endl << + endl; + } + + indent_down(); + f_header_ << + indent() << "};" << endl << + endl; +} + +/** + * Generates a service client definition. + * + * @param tservice The service to generate a server for. + */ +void t_cpp_generator::generate_service_client(t_service* tservice) { + string extends = ""; + string extends_client = ""; + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + extends_client = ", public " + extends + "Client"; + } + + // Generate the header portion + f_header_ << + "class " << service_name_ << "Client : " << + "virtual public " << service_name_ << "If" << + extends_client << " {" << endl << + " public:" << endl; + + indent_up(); + f_header_ << + indent() << service_name_ << "Client(boost::shared_ptr prot) :" << endl; + if (extends.empty()) { + f_header_ << + indent() << " piprot_(prot)," << endl << + indent() << " poprot_(prot) {" << endl << + indent() << " iprot_ = prot.get();" << endl << + indent() << " oprot_ = prot.get();" << endl << + indent() << "}" << endl; + } else { + f_header_ << + indent() << " " << extends << "Client(prot, prot) {}" << endl; + } + + f_header_ << + indent() << service_name_ << "Client(boost::shared_ptr iprot, boost::shared_ptr oprot) :" << endl; + if (extends.empty()) { + f_header_ << + indent() << " piprot_(iprot)," << endl << + indent() << " poprot_(oprot) {" << endl << + indent() << " iprot_ = iprot.get();" << endl << + indent() << " oprot_ = oprot.get();" << endl << + indent() << "}" << endl; + } else { + f_header_ << + indent() << " " << extends << "Client(iprot, oprot) {}" << endl; + } + + // Generate getters for the protocols. + f_header_ << + indent() << "boost::shared_ptr getInputProtocol() {" << endl << + indent() << " return piprot_;" << endl << + indent() << "}" << endl; + + f_header_ << + indent() << "boost::shared_ptr getOutputProtocol() {" << endl << + indent() << " return poprot_;" << endl << + indent() << "}" << endl; + + vector functions = tservice->get_functions(); + vector::const_iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_function send_function(g_type_void, + string("send_") + (*f_iter)->get_name(), + (*f_iter)->get_arglist()); + indent(f_header_) << function_signature(*f_iter) << ";" << endl; + indent(f_header_) << function_signature(&send_function) << ";" << endl; + if (!(*f_iter)->is_oneway()) { + t_struct noargs(program_); + t_function recv_function((*f_iter)->get_returntype(), + string("recv_") + (*f_iter)->get_name(), + &noargs); + indent(f_header_) << function_signature(&recv_function) << ";" << endl; + } + } + indent_down(); + + if (extends.empty()) { + f_header_ << + " protected:" << endl; + indent_up(); + f_header_ << + indent() << "boost::shared_ptr piprot_;" << endl << + indent() << "boost::shared_ptr poprot_;" << endl << + indent() << "apache::thrift::protocol::TProtocol* iprot_;" << endl << + indent() << "apache::thrift::protocol::TProtocol* oprot_;" << endl; + indent_down(); + } + + f_header_ << + "};" << endl << + endl; + + string scope = service_name_ + "Client::"; + + // Generate client method implementations + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string funname = (*f_iter)->get_name(); + + // Open function + indent(f_service_) << + function_signature(*f_iter, scope) << endl; + scope_up(f_service_); + indent(f_service_) << + "send_" << funname << "("; + + // Get the struct of function call params + t_struct* arg_struct = (*f_iter)->get_arglist(); + + // Declare the function arguments + const vector& fields = arg_struct->get_members(); + vector::const_iterator fld_iter; + bool first = true; + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << (*fld_iter)->get_name(); + } + f_service_ << ");" << endl; + + if (!(*f_iter)->is_oneway()) { + f_service_ << indent(); + if (!(*f_iter)->get_returntype()->is_void()) { + if (is_complex_type((*f_iter)->get_returntype())) { + f_service_ << "recv_" << funname << "(_return);" << endl; + } else { + f_service_ << "return recv_" << funname << "();" << endl; + } + } else { + f_service_ << + "recv_" << funname << "();" << endl; + } + } + scope_down(f_service_); + f_service_ << endl; + + // Function for sending + t_function send_function(g_type_void, + string("send_") + (*f_iter)->get_name(), + (*f_iter)->get_arglist()); + + // Open the send function + indent(f_service_) << + function_signature(&send_function, scope) << endl; + scope_up(f_service_); + + // Function arguments and results + string argsname = tservice->get_name() + "_" + (*f_iter)->get_name() + "_pargs"; + string resultname = tservice->get_name() + "_" + (*f_iter)->get_name() + "_presult"; + + // Serialize the request + f_service_ << + indent() << "int32_t cseqid = 0;" << endl << + indent() << "oprot_->writeMessageBegin(\"" << (*f_iter)->get_name() << "\", apache::thrift::protocol::T_CALL, cseqid);" << endl << + endl << + indent() << argsname << " args;" << endl; + + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + f_service_ << + indent() << "args." << (*fld_iter)->get_name() << " = &" << (*fld_iter)->get_name() << ";" << endl; + } + + f_service_ << + indent() << "args.write(oprot_);" << endl << + endl << + indent() << "oprot_->writeMessageEnd();" << endl << + indent() << "oprot_->getTransport()->flush();" << endl << + indent() << "oprot_->getTransport()->writeEnd();" << endl; + + scope_down(f_service_); + f_service_ << endl; + + // Generate recv function only if not an oneway function + if (!(*f_iter)->is_oneway()) { + t_struct noargs(program_); + t_function recv_function((*f_iter)->get_returntype(), + string("recv_") + (*f_iter)->get_name(), + &noargs); + // Open function + indent(f_service_) << + function_signature(&recv_function, scope) << endl; + scope_up(f_service_); + + f_service_ << + endl << + indent() << "int32_t rseqid = 0;" << endl << + indent() << "std::string fname;" << endl << + indent() << "apache::thrift::protocol::TMessageType mtype;" << endl << + endl << + indent() << "iprot_->readMessageBegin(fname, mtype, rseqid);" << endl << + indent() << "if (mtype == apache::thrift::protocol::T_EXCEPTION) {" << endl << + indent() << " apache::thrift::TApplicationException x;" << endl << + indent() << " x.read(iprot_);" << endl << + indent() << " iprot_->readMessageEnd();" << endl << + indent() << " iprot_->getTransport()->readEnd();" << endl << + indent() << " throw x;" << endl << + indent() << "}" << endl << + indent() << "if (mtype != apache::thrift::protocol::T_REPLY) {" << endl << + indent() << " iprot_->skip(apache::thrift::protocol::T_STRUCT);" << endl << + indent() << " iprot_->readMessageEnd();" << endl << + indent() << " iprot_->getTransport()->readEnd();" << endl << + indent() << " throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE);" << endl << + indent() << "}" << endl << + indent() << "if (fname.compare(\"" << (*f_iter)->get_name() << "\") != 0) {" << endl << + indent() << " iprot_->skip(apache::thrift::protocol::T_STRUCT);" << endl << + indent() << " iprot_->readMessageEnd();" << endl << + indent() << " iprot_->getTransport()->readEnd();" << endl << + indent() << " throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::WRONG_METHOD_NAME);" << endl << + indent() << "}" << endl; + + if (!(*f_iter)->get_returntype()->is_void() && + !is_complex_type((*f_iter)->get_returntype())) { + t_field returnfield((*f_iter)->get_returntype(), "_return"); + f_service_ << + indent() << declare_field(&returnfield) << endl; + } + + f_service_ << + indent() << resultname << " result;" << endl; + + if (!(*f_iter)->get_returntype()->is_void()) { + f_service_ << + indent() << "result.success = &_return;" << endl; + } + + f_service_ << + indent() << "result.read(iprot_);" << endl << + indent() << "iprot_->readMessageEnd();" << endl << + indent() << "iprot_->getTransport()->readEnd();" << endl << + endl; + + // Careful, only look for _result if not a void function + if (!(*f_iter)->get_returntype()->is_void()) { + if (is_complex_type((*f_iter)->get_returntype())) { + f_service_ << + indent() << "if (result.__isset.success) {" << endl << + indent() << " // _return pointer has now been filled" << endl << + indent() << " return;" << endl << + indent() << "}" << endl; + } else { + f_service_ << + indent() << "if (result.__isset.success) {" << endl << + indent() << " return _return;" << endl << + indent() << "}" << endl; + } + } + + t_struct* xs = (*f_iter)->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << + indent() << "if (result.__isset." << (*x_iter)->get_name() << ") {" << endl << + indent() << " throw result." << (*x_iter)->get_name() << ";" << endl << + indent() << "}" << endl; + } + + // We only get here if we are a void function + if ((*f_iter)->get_returntype()->is_void()) { + indent(f_service_) << + "return;" << endl; + } else { + f_service_ << + indent() << "throw apache::thrift::TApplicationException(apache::thrift::TApplicationException::MISSING_RESULT, \"" << (*f_iter)->get_name() << " failed: unknown result\");" << endl; + } + + // Close function + scope_down(f_service_); + f_service_ << endl; + } + } +} + +/** + * Generates a service server definition. + * + * @param tservice The service to generate a server for. + */ +void t_cpp_generator::generate_service_processor(t_service* tservice) { + // Generate the dispatch methods + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + string extends = ""; + string extends_processor = ""; + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + extends_processor = ", public " + extends + "Processor"; + } + + // Generate the header portion + f_header_ << + "class " << service_name_ << "Processor : " << + "virtual public apache::thrift::TProcessor" << + extends_processor << " {" << endl; + + // Protected data members + f_header_ << + " protected:" << endl; + indent_up(); + f_header_ << + indent() << "boost::shared_ptr<" << service_name_ << "If> iface_;" << endl; + f_header_ << + indent() << "virtual bool process_fn(apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, std::string& fname, int32_t seqid);" << endl; + indent_down(); + + // Process function declarations + f_header_ << + " private:" << endl; + indent_up(); + f_header_ << + indent() << "std::map processMap_;" << endl; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + indent(f_header_) << + "void process_" << (*f_iter)->get_name() << "(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot);" << endl; + } + indent_down(); + + indent_up(); + string declare_map = ""; + indent_up(); + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + declare_map += indent(); + declare_map += "processMap_[\""; + declare_map += (*f_iter)->get_name(); + declare_map += "\"] = &"; + declare_map += service_name_; + declare_map += "Processor::process_"; + declare_map += (*f_iter)->get_name(); + declare_map += ";\n"; + } + indent_down(); + + f_header_ << + " public:" << endl << + indent() << service_name_ << "Processor(boost::shared_ptr<" << service_name_ << "If> iface) :" << endl; + if (extends.empty()) { + f_header_ << + indent() << " iface_(iface) {" << endl; + } else { + f_header_ << + indent() << " " << extends << "Processor(iface)," << endl << + indent() << " iface_(iface) {" << endl; + } + f_header_ << + declare_map << + indent() << "}" << endl << + endl << + indent() << "virtual bool process(boost::shared_ptr piprot, boost::shared_ptr poprot);" << endl << + indent() << "virtual ~" << service_name_ << "Processor() {}" << endl; + indent_down(); + f_header_ << + "};" << endl << endl; + + // Generate the server implementation + f_service_ << + "bool " << service_name_ << "Processor::process(boost::shared_ptr piprot, boost::shared_ptr poprot) {" << endl; + indent_up(); + + f_service_ << + endl << + indent() << "apache::thrift::protocol::TProtocol* iprot = piprot.get();" << endl << + indent() << "apache::thrift::protocol::TProtocol* oprot = poprot.get();" << endl << + indent() << "std::string fname;" << endl << + indent() << "apache::thrift::protocol::TMessageType mtype;" << endl << + indent() << "int32_t seqid;" << endl << + endl << + indent() << "iprot->readMessageBegin(fname, mtype, seqid);" << endl << + endl << + indent() << "if (mtype != apache::thrift::protocol::T_CALL && mtype != apache::thrift::protocol::T_ONEWAY) {" << endl << + indent() << " iprot->skip(apache::thrift::protocol::T_STRUCT);" << endl << + indent() << " iprot->readMessageEnd();" << endl << + indent() << " iprot->getTransport()->readEnd();" << endl << + indent() << " apache::thrift::TApplicationException x(apache::thrift::TApplicationException::INVALID_MESSAGE_TYPE);" << endl << + indent() << " oprot->writeMessageBegin(fname, apache::thrift::protocol::T_EXCEPTION, seqid);" << endl << + indent() << " x.write(oprot);" << endl << + indent() << " oprot->writeMessageEnd();" << endl << + indent() << " oprot->getTransport()->flush();" << endl << + indent() << " oprot->getTransport()->writeEnd();" << endl << + indent() << " return true;" << endl << + indent() << "}" << endl << + endl << + indent() << "return process_fn(iprot, oprot, fname, seqid);" << + endl; + + indent_down(); + f_service_ << + indent() << "}" << endl << + endl; + + f_service_ << + "bool " << service_name_ << "Processor::process_fn(apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot, std::string& fname, int32_t seqid) {" << endl; + indent_up(); + + // HOT: member function pointer map + f_service_ << + indent() << "std::map::iterator pfn;" << endl << + indent() << "pfn = processMap_.find(fname);" << endl << + indent() << "if (pfn == processMap_.end()) {" << endl; + if (extends.empty()) { + f_service_ << + indent() << " iprot->skip(apache::thrift::protocol::T_STRUCT);" << endl << + indent() << " iprot->readMessageEnd();" << endl << + indent() << " iprot->getTransport()->readEnd();" << endl << + indent() << " apache::thrift::TApplicationException x(apache::thrift::TApplicationException::UNKNOWN_METHOD, \"Invalid method name: '\"+fname+\"'\");" << endl << + indent() << " oprot->writeMessageBegin(fname, apache::thrift::protocol::T_EXCEPTION, seqid);" << endl << + indent() << " x.write(oprot);" << endl << + indent() << " oprot->writeMessageEnd();" << endl << + indent() << " oprot->getTransport()->flush();" << endl << + indent() << " oprot->getTransport()->writeEnd();" << endl << + indent() << " return true;" << endl; + } else { + f_service_ << + indent() << " return " << extends << "Processor::process_fn(iprot, oprot, fname, seqid);" << endl; + } + f_service_ << + indent() << "}" << endl << + indent() << "(this->*(pfn->second))(seqid, iprot, oprot);" << endl << + indent() << "return true;" << endl; + + indent_down(); + f_service_ << + "}" << endl << + endl; + + // Generate the process subfunctions + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + generate_process_function(tservice, *f_iter); + } +} + +/** + * Generates a struct and helpers for a function. + * + * @param tfunction The function + */ +void t_cpp_generator::generate_function_helpers(t_service* tservice, + t_function* tfunction) { + if (tfunction->is_oneway()) { + return; + } + + t_struct result(program_, tservice->get_name() + "_" + tfunction->get_name() + "_result"); + t_field success(tfunction->get_returntype(), "success", 0); + if (!tfunction->get_returntype()->is_void()) { + result.append(&success); + } + + t_struct* xs = tfunction->get_xceptions(); + const vector& fields = xs->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + result.append(*f_iter); + } + + generate_struct_definition(f_header_, &result, false); + generate_struct_reader(f_service_, &result); + generate_struct_result_writer(f_service_, &result); + + result.set_name(tservice->get_name() + "_" + tfunction->get_name() + "_presult"); + generate_struct_definition(f_header_, &result, false, true, true, false); + generate_struct_reader(f_service_, &result, true); + +} + +/** + * Generates a process function definition. + * + * @param tfunction The function to write a dispatcher for + */ +void t_cpp_generator::generate_process_function(t_service* tservice, + t_function* tfunction) { + // Open function + f_service_ << + "void " << tservice->get_name() << "Processor::" << + "process_" << tfunction->get_name() << + "(int32_t seqid, apache::thrift::protocol::TProtocol* iprot, apache::thrift::protocol::TProtocol* oprot)" << endl; + scope_up(f_service_); + + string argsname = tservice->get_name() + "_" + tfunction->get_name() + "_args"; + string resultname = tservice->get_name() + "_" + tfunction->get_name() + "_result"; + + f_service_ << + indent() << argsname << " args;" << endl << + indent() << "args.read(iprot);" << endl << + indent() << "iprot->readMessageEnd();" << endl << + indent() << "iprot->getTransport()->readEnd();" << endl << + endl; + + t_struct* xs = tfunction->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + + // Declare result + if (!tfunction->is_oneway()) { + f_service_ << + indent() << resultname << " result;" << endl; + } + + // Try block for functions with exceptions + f_service_ << + indent() << "try {" << endl; + indent_up(); + + // Generate the function call + t_struct* arg_struct = tfunction->get_arglist(); + const std::vector& fields = arg_struct->get_members(); + vector::const_iterator f_iter; + + bool first = true; + f_service_ << indent(); + if (!tfunction->is_oneway() && !tfunction->get_returntype()->is_void()) { + if (is_complex_type(tfunction->get_returntype())) { + first = false; + f_service_ << "iface_->" << tfunction->get_name() << "(result.success"; + } else { + f_service_ << "result.success = iface_->" << tfunction->get_name() << "("; + } + } else { + f_service_ << + "iface_->" << tfunction->get_name() << "("; + } + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << "args." << (*f_iter)->get_name(); + } + f_service_ << ");" << endl; + + // Set isset on success field + if (!tfunction->is_oneway() && !tfunction->get_returntype()->is_void()) { + f_service_ << + indent() << "result.__isset.success = true;" << endl; + } + + indent_down(); + f_service_ << indent() << "}"; + + if (!tfunction->is_oneway()) { + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << " catch (" << type_name((*x_iter)->get_type()) << " &" << (*x_iter)->get_name() << ") {" << endl; + if (!tfunction->is_oneway()) { + indent_up(); + f_service_ << + indent() << "result." << (*x_iter)->get_name() << " = " << (*x_iter)->get_name() << ";" << endl << + indent() << "result.__isset." << (*x_iter)->get_name() << " = true;" << endl; + indent_down(); + f_service_ << indent() << "}"; + } else { + f_service_ << "}"; + } + } + } + + f_service_ << " catch (const std::exception& e) {" << endl; + + if (!tfunction->is_oneway()) { + indent_up(); + f_service_ << + indent() << "apache::thrift::TApplicationException x(e.what());" << endl << + indent() << "oprot->writeMessageBegin(\"" << tfunction->get_name() << "\", apache::thrift::protocol::T_EXCEPTION, seqid);" << endl << + indent() << "x.write(oprot);" << endl << + indent() << "oprot->writeMessageEnd();" << endl << + indent() << "oprot->getTransport()->flush();" << endl << + indent() << "oprot->getTransport()->writeEnd();" << endl << + indent() << "return;" << endl; + indent_down(); + } + f_service_ << indent() << "}" << endl; + + // Shortcut out here for oneway functions + if (tfunction->is_oneway()) { + f_service_ << + indent() << "return;" << endl; + indent_down(); + f_service_ << "}" << endl << + endl; + return; + } + + // Serialize the result into a struct + f_service_ << + endl << + indent() << "oprot->writeMessageBegin(\"" << tfunction->get_name() << "\", apache::thrift::protocol::T_REPLY, seqid);" << endl << + indent() << "result.write(oprot);" << endl << + indent() << "oprot->writeMessageEnd();" << endl << + indent() << "oprot->getTransport()->flush();" << endl << + indent() << "oprot->getTransport()->writeEnd();" << endl; + + // Close function + scope_down(f_service_); + f_service_ << endl; +} + +/** + * Generates a skeleton file of a server + * + * @param tservice The service to generate a server for. + */ +void t_cpp_generator::generate_service_skeleton(t_service* tservice) { + string svcname = tservice->get_name(); + + // Service implementation file includes + string f_skeleton_name = get_out_dir()+svcname+"_server.skeleton.cpp"; + + string ns = namespace_prefix(tservice->get_program()->get_namespace("cpp")); + + ofstream f_skeleton; + f_skeleton.open(f_skeleton_name.c_str()); + f_skeleton << + "// This autogenerated skeleton file illustrates how to build a server." << endl << + "// You should copy it to another filename to avoid overwriting it." << endl << + endl << + "#include \"" << get_include_prefix(*get_program()) << svcname << ".h\"" << endl << + "#include " << endl << + "#include " << endl << + "#include " << endl << + "#include " << endl << + endl << + "using namespace apache::thrift;" << endl << + "using namespace apache::thrift::protocol;" << endl << + "using namespace apache::thrift::transport;" << endl << + "using namespace apache::thrift::server;" << endl << + endl << + "using boost::shared_ptr;" << endl << + endl; + + if (!ns.empty()) { + f_skeleton << + "using namespace " << string(ns, 0, ns.size()-2) << ";" << endl << + endl; + } + + f_skeleton << + "class " << svcname << "Handler : virtual public " << svcname << "If {" << endl << + " public:" << endl; + indent_up(); + f_skeleton << + indent() << svcname << "Handler() {" << endl << + indent() << " // Your initialization goes here" << endl << + indent() << "}" << endl << + endl; + + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_skeleton << + indent() << function_signature(*f_iter) << " {" << endl << + indent() << " // Your implementation goes here" << endl << + indent() << " printf(\"" << (*f_iter)->get_name() << "\\n\");" << endl << + indent() << "}" << endl << + endl; + } + + indent_down(); + f_skeleton << + "};" << endl << + endl; + + f_skeleton << + indent() << "int main(int argc, char **argv) {" << endl; + indent_up(); + f_skeleton << + indent() << "int port = 9090;" << endl << + indent() << "shared_ptr<" << svcname << "Handler> handler(new " << svcname << "Handler());" << endl << + indent() << "shared_ptr processor(new " << svcname << "Processor(handler));" << endl << + indent() << "shared_ptr serverTransport(new TServerSocket(port));" << endl << + indent() << "shared_ptr transportFactory(new TBufferedTransportFactory());" << endl << + indent() << "shared_ptr protocolFactory(new TBinaryProtocolFactory());" << endl << + endl << + indent() << "TSimpleServer server(processor, serverTransport, transportFactory, protocolFactory);" << endl << + indent() << "server.serve();" << endl << + indent() << "return 0;" << endl; + indent_down(); + f_skeleton << + "}" << endl << + endl; + + // Close the files + f_skeleton.close(); +} + +/** + * Deserializes a field of any type. + */ +void t_cpp_generator::generate_deserialize_field(ofstream& out, + t_field* tfield, + string prefix, + string suffix) { + t_type* type = get_true_type(tfield->get_type()); + + if (type->is_void()) { + throw "CANNOT GENERATE DESERIALIZE CODE FOR void TYPE: " + + prefix + tfield->get_name(); + } + + string name = prefix + tfield->get_name() + suffix; + + if (type->is_struct() || type->is_xception()) { + generate_deserialize_struct(out, (t_struct*)type, name); + } else if (type->is_container()) { + generate_deserialize_container(out, type, name); + } else if (type->is_base_type()) { + indent(out) << + "xfer += iprot->"; + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "compiler error: cannot serialize void field in a struct: " + name; + break; + case t_base_type::TYPE_STRING: + if (((t_base_type*)type)->is_binary()) { + out << "readBinary(" << name << ");"; + } + else { + out << "readString(" << name << ");"; + } + break; + case t_base_type::TYPE_BOOL: + out << "readBool(" << name << ");"; + break; + case t_base_type::TYPE_BYTE: + out << "readByte(" << name << ");"; + break; + case t_base_type::TYPE_I16: + out << "readI16(" << name << ");"; + break; + case t_base_type::TYPE_I32: + out << "readI32(" << name << ");"; + break; + case t_base_type::TYPE_I64: + out << "readI64(" << name << ");"; + break; + case t_base_type::TYPE_DOUBLE: + out << "readDouble(" << name << ");"; + break; + default: + throw "compiler error: no C++ reader for base type " + t_base_type::t_base_name(tbase) + name; + } + out << + endl; + } else if (type->is_enum()) { + string t = tmp("ecast"); + out << + indent() << "int32_t " << t << ";" << endl << + indent() << "xfer += iprot->readI32(" << t << ");" << endl << + indent() << name << " = (" << type_name(type) << ")" << t << ";" << endl; + } else { + printf("DO NOT KNOW HOW TO DESERIALIZE FIELD '%s' TYPE '%s'\n", + tfield->get_name().c_str(), type_name(type).c_str()); + } +} + +/** + * Generates an unserializer for a variable. This makes two key assumptions, + * first that there is a const char* variable named data that points to the + * buffer for deserialization, and that there is a variable protocol which + * is a reference to a TProtocol serialization object. + */ +void t_cpp_generator::generate_deserialize_struct(ofstream& out, + t_struct* tstruct, + string prefix) { + indent(out) << + "xfer += " << prefix << ".read(iprot);" << endl; +} + +void t_cpp_generator::generate_deserialize_container(ofstream& out, + t_type* ttype, + string prefix) { + scope_up(out); + + string size = tmp("_size"); + string ktype = tmp("_ktype"); + string vtype = tmp("_vtype"); + string etype = tmp("_etype"); + + t_container* tcontainer = (t_container*)ttype; + bool use_push = tcontainer->has_cpp_name(); + + indent(out) << + prefix << ".clear();" << endl << + indent() << "uint32_t " << size << ";" << endl; + + // Declare variables, read header + if (ttype->is_map()) { + out << + indent() << "apache::thrift::protocol::TType " << ktype << ";" << endl << + indent() << "apache::thrift::protocol::TType " << vtype << ";" << endl << + indent() << "iprot->readMapBegin(" << + ktype << ", " << vtype << ", " << size << ");" << endl; + } else if (ttype->is_set()) { + out << + indent() << "apache::thrift::protocol::TType " << etype << ";" << endl << + indent() << "iprot->readSetBegin(" << + etype << ", " << size << ");" << endl; + } else if (ttype->is_list()) { + out << + indent() << "apache::thrift::protocol::TType " << etype << ";" << endl << + indent() << "iprot->readListBegin(" << + etype << ", " << size << ");" << endl; + if (!use_push) { + indent(out) << prefix << ".resize(" << size << ");" << endl; + } + } + + + // For loop iterates over elements + string i = tmp("_i"); + out << + indent() << "uint32_t " << i << ";" << endl << + indent() << "for (" << i << " = 0; " << i << " < " << size << "; ++" << i << ")" << endl; + + scope_up(out); + + if (ttype->is_map()) { + generate_deserialize_map_element(out, (t_map*)ttype, prefix); + } else if (ttype->is_set()) { + generate_deserialize_set_element(out, (t_set*)ttype, prefix); + } else if (ttype->is_list()) { + generate_deserialize_list_element(out, (t_list*)ttype, prefix, use_push, i); + } + + scope_down(out); + + // Read container end + if (ttype->is_map()) { + indent(out) << "iprot->readMapEnd();" << endl; + } else if (ttype->is_set()) { + indent(out) << "iprot->readSetEnd();" << endl; + } else if (ttype->is_list()) { + indent(out) << "iprot->readListEnd();" << endl; + } + + scope_down(out); +} + + +/** + * Generates code to deserialize a map + */ +void t_cpp_generator::generate_deserialize_map_element(ofstream& out, + t_map* tmap, + string prefix) { + string key = tmp("_key"); + string val = tmp("_val"); + t_field fkey(tmap->get_key_type(), key); + t_field fval(tmap->get_val_type(), val); + + out << + indent() << declare_field(&fkey) << endl; + + generate_deserialize_field(out, &fkey); + indent(out) << + declare_field(&fval, false, false, false, true) << " = " << + prefix << "[" << key << "];" << endl; + + generate_deserialize_field(out, &fval); +} + +void t_cpp_generator::generate_deserialize_set_element(ofstream& out, + t_set* tset, + string prefix) { + string elem = tmp("_elem"); + t_field felem(tset->get_elem_type(), elem); + + indent(out) << + declare_field(&felem) << endl; + + generate_deserialize_field(out, &felem); + + indent(out) << + prefix << ".insert(" << elem << ");" << endl; +} + +void t_cpp_generator::generate_deserialize_list_element(ofstream& out, + t_list* tlist, + string prefix, + bool use_push, + string index) { + if (use_push) { + string elem = tmp("_elem"); + t_field felem(tlist->get_elem_type(), elem); + indent(out) << declare_field(&felem) << endl; + generate_deserialize_field(out, &felem); + indent(out) << prefix << ".push_back(" << elem << ");" << endl; + } else { + t_field felem(tlist->get_elem_type(), prefix + "[" + index + "]"); + generate_deserialize_field(out, &felem); + } +} + + +/** + * Serializes a field of any type. + * + * @param tfield The field to serialize + * @param prefix Name to prepend to field name + */ +void t_cpp_generator::generate_serialize_field(ofstream& out, + t_field* tfield, + string prefix, + string suffix) { + t_type* type = get_true_type(tfield->get_type()); + + string name = prefix + tfield->get_name() + suffix; + + // Do nothing for void types + if (type->is_void()) { + throw "CANNOT GENERATE SERIALIZE CODE FOR void TYPE: " + name; + } + + + + if (type->is_struct() || type->is_xception()) { + generate_serialize_struct(out, + (t_struct*)type, + name); + } else if (type->is_container()) { + generate_serialize_container(out, type, name); + } else if (type->is_base_type() || type->is_enum()) { + + indent(out) << + "xfer += oprot->"; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw + "compiler error: cannot serialize void field in a struct: " + name; + break; + case t_base_type::TYPE_STRING: + if (((t_base_type*)type)->is_binary()) { + out << "writeBinary(" << name << ");"; + } + else { + out << "writeString(" << name << ");"; + } + break; + case t_base_type::TYPE_BOOL: + out << "writeBool(" << name << ");"; + break; + case t_base_type::TYPE_BYTE: + out << "writeByte(" << name << ");"; + break; + case t_base_type::TYPE_I16: + out << "writeI16(" << name << ");"; + break; + case t_base_type::TYPE_I32: + out << "writeI32(" << name << ");"; + break; + case t_base_type::TYPE_I64: + out << "writeI64(" << name << ");"; + break; + case t_base_type::TYPE_DOUBLE: + out << "writeDouble(" << name << ");"; + break; + default: + throw "compiler error: no C++ writer for base type " + t_base_type::t_base_name(tbase) + name; + } + } else if (type->is_enum()) { + out << "writeI32((int32_t)" << name << ");"; + } + out << endl; + } else { + printf("DO NOT KNOW HOW TO SERIALIZE FIELD '%s' TYPE '%s'\n", + name.c_str(), + type_name(type).c_str()); + } +} + +/** + * Serializes all the members of a struct. + * + * @param tstruct The struct to serialize + * @param prefix String prefix to attach to all fields + */ +void t_cpp_generator::generate_serialize_struct(ofstream& out, + t_struct* tstruct, + string prefix) { + indent(out) << + "xfer += " << prefix << ".write(oprot);" << endl; +} + +void t_cpp_generator::generate_serialize_container(ofstream& out, + t_type* ttype, + string prefix) { + scope_up(out); + + if (ttype->is_map()) { + indent(out) << + "xfer += oprot->writeMapBegin(" << + type_to_enum(((t_map*)ttype)->get_key_type()) << ", " << + type_to_enum(((t_map*)ttype)->get_val_type()) << ", " << + prefix << ".size());" << endl; + } else if (ttype->is_set()) { + indent(out) << + "xfer += oprot->writeSetBegin(" << + type_to_enum(((t_set*)ttype)->get_elem_type()) << ", " << + prefix << ".size());" << endl; + } else if (ttype->is_list()) { + indent(out) << + "xfer += oprot->writeListBegin(" << + type_to_enum(((t_list*)ttype)->get_elem_type()) << ", " << + prefix << ".size());" << endl; + } + + string iter = tmp("_iter"); + out << + indent() << type_name(ttype) << "::const_iterator " << iter << ";" << endl << + indent() << "for (" << iter << " = " << prefix << ".begin(); " << iter << " != " << prefix << ".end(); ++" << iter << ")" << endl; + scope_up(out); + if (ttype->is_map()) { + generate_serialize_map_element(out, (t_map*)ttype, iter); + } else if (ttype->is_set()) { + generate_serialize_set_element(out, (t_set*)ttype, iter); + } else if (ttype->is_list()) { + generate_serialize_list_element(out, (t_list*)ttype, iter); + } + scope_down(out); + + if (ttype->is_map()) { + indent(out) << + "xfer += oprot->writeMapEnd();" << endl; + } else if (ttype->is_set()) { + indent(out) << + "xfer += oprot->writeSetEnd();" << endl; + } else if (ttype->is_list()) { + indent(out) << + "xfer += oprot->writeListEnd();" << endl; + } + + scope_down(out); +} + +/** + * Serializes the members of a map. + * + */ +void t_cpp_generator::generate_serialize_map_element(ofstream& out, + t_map* tmap, + string iter) { + t_field kfield(tmap->get_key_type(), iter + "->first"); + generate_serialize_field(out, &kfield, ""); + + t_field vfield(tmap->get_val_type(), iter + "->second"); + generate_serialize_field(out, &vfield, ""); +} + +/** + * Serializes the members of a set. + */ +void t_cpp_generator::generate_serialize_set_element(ofstream& out, + t_set* tset, + string iter) { + t_field efield(tset->get_elem_type(), "(*" + iter + ")"); + generate_serialize_field(out, &efield, ""); +} + +/** + * Serializes the members of a list. + */ +void t_cpp_generator::generate_serialize_list_element(ofstream& out, + t_list* tlist, + string iter) { + t_field efield(tlist->get_elem_type(), "(*" + iter + ")"); + generate_serialize_field(out, &efield, ""); +} + +/** + * Makes a :: prefix for a namespace + * + * @param ns The namepsace, w/ periods in it + * @return Namespaces + */ +string t_cpp_generator::namespace_prefix(string ns) { + if (ns.size() == 0) { + return ""; + } + string result = ""; + string::size_type loc; + while ((loc = ns.find(".")) != string::npos) { + result += ns.substr(0, loc); + result += "::"; + ns = ns.substr(loc+1); + } + if (ns.size() > 0) { + result += ns + "::"; + } + return result; +} + +/** + * Opens namespace. + * + * @param ns The namepsace, w/ periods in it + * @return Namespaces + */ +string t_cpp_generator::namespace_open(string ns) { + if (ns.size() == 0) { + return ""; + } + string result = ""; + string separator = ""; + string::size_type loc; + while ((loc = ns.find(".")) != string::npos) { + result += separator; + result += "namespace "; + result += ns.substr(0, loc); + result += " {"; + separator = " "; + ns = ns.substr(loc+1); + } + if (ns.size() > 0) { + result += separator + "namespace " + ns + " {"; + } + return result; +} + +/** + * Closes namespace. + * + * @param ns The namepsace, w/ periods in it + * @return Namespaces + */ +string t_cpp_generator::namespace_close(string ns) { + if (ns.size() == 0) { + return ""; + } + string result = "}"; + string::size_type loc; + while ((loc = ns.find(".")) != string::npos) { + result += "}"; + ns = ns.substr(loc+1); + } + result += " // namespace"; + return result; +} + +/** + * Returns a C++ type name + * + * @param ttype The type + * @return String of the type name, i.e. std::set + */ +string t_cpp_generator::type_name(t_type* ttype, bool in_typedef, bool arg) { + if (ttype->is_base_type()) { + string bname = base_type_name(((t_base_type*)ttype)->get_base()); + if (!arg) { + return bname; + } + + if (((t_base_type*)ttype)->get_base() == t_base_type::TYPE_STRING) { + return "const " + bname + "&"; + } else { + return "const " + bname; + } + } + + // Check for a custom overloaded C++ name + if (ttype->is_container()) { + string cname; + + t_container* tcontainer = (t_container*) ttype; + if (tcontainer->has_cpp_name()) { + cname = tcontainer->get_cpp_name(); + } else if (ttype->is_map()) { + t_map* tmap = (t_map*) ttype; + cname = "std::map<" + + type_name(tmap->get_key_type(), in_typedef) + ", " + + type_name(tmap->get_val_type(), in_typedef) + "> "; + } else if (ttype->is_set()) { + t_set* tset = (t_set*) ttype; + cname = "std::set<" + type_name(tset->get_elem_type(), in_typedef) + "> "; + } else if (ttype->is_list()) { + t_list* tlist = (t_list*) ttype; + cname = "std::vector<" + type_name(tlist->get_elem_type(), in_typedef) + "> "; + } + + if (arg) { + return "const " + cname + "&"; + } else { + return cname; + } + } + + string class_prefix; + if (in_typedef && (ttype->is_struct() || ttype->is_xception())) { + class_prefix = "class "; + } + + // Check if it needs to be namespaced + string pname; + t_program* program = ttype->get_program(); + if (program != NULL && program != program_) { + pname = + class_prefix + + namespace_prefix(program->get_namespace("cpp")) + + ttype->get_name(); + } else { + pname = class_prefix + ttype->get_name(); + } + + if (arg) { + if (is_complex_type(ttype)) { + return "const " + pname + "&"; + } else { + return "const " + pname; + } + } else { + return pname; + } +} + +/** + * Returns the C++ type that corresponds to the thrift type. + * + * @param tbase The base type + * @return Explicit C++ type, i.e. "int32_t" + */ +string t_cpp_generator::base_type_name(t_base_type::t_base tbase) { + switch (tbase) { + case t_base_type::TYPE_VOID: + return "void"; + case t_base_type::TYPE_STRING: + return "std::string"; + case t_base_type::TYPE_BOOL: + return "bool"; + case t_base_type::TYPE_BYTE: + return "int8_t"; + case t_base_type::TYPE_I16: + return "int16_t"; + case t_base_type::TYPE_I32: + return "int32_t"; + case t_base_type::TYPE_I64: + return "int64_t"; + case t_base_type::TYPE_DOUBLE: + return "double"; + default: + throw "compiler error: no C++ base type name for base type " + t_base_type::t_base_name(tbase); + } +} + +/** + * Declares a field, which may include initialization as necessary. + * + * @param ttype The type + * @return Field declaration, i.e. int x = 0; + */ +string t_cpp_generator::declare_field(t_field* tfield, bool init, bool pointer, bool constant, bool reference) { + // TODO(mcslee): do we ever need to initialize the field? + string result = ""; + if (constant) { + result += "const "; + } + result += type_name(tfield->get_type()); + if (pointer) { + result += "*"; + } + if (reference) { + result += "&"; + } + result += " " + tfield->get_name(); + if (init) { + t_type* type = get_true_type(tfield->get_type()); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + break; + case t_base_type::TYPE_STRING: + result += " = \"\""; + break; + case t_base_type::TYPE_BOOL: + result += " = false"; + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + result += " = 0"; + break; + case t_base_type::TYPE_DOUBLE: + result += " = (double)0"; + break; + default: + throw "compiler error: no C++ initializer for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + result += " = (" + type_name(type) + ")0"; + } + } + if (!reference) { + result += ";"; + } + return result; +} + +/** + * Renders a function signature of the form 'type name(args)' + * + * @param tfunction Function definition + * @return String of rendered function definition + */ +string t_cpp_generator::function_signature(t_function* tfunction, + string prefix, + bool name_params) { + t_type* ttype = tfunction->get_returntype(); + t_struct* arglist = tfunction->get_arglist(); + + if (is_complex_type(ttype)) { + bool empty = arglist->get_members().size() == 0; + return + "void " + prefix + tfunction->get_name() + + "(" + type_name(ttype) + (name_params ? "& _return" : "& /* _return */") + + (empty ? "" : (", " + argument_list(arglist, name_params))) + ")"; + } else { + return + type_name(ttype) + " " + prefix + tfunction->get_name() + + "(" + argument_list(arglist, name_params) + ")"; + } +} + +/** + * Renders a field list + * + * @param tstruct The struct definition + * @return Comma sepearated list of all field names in that struct + */ +string t_cpp_generator::argument_list(t_struct* tstruct, bool name_params) { + string result = ""; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + result += ", "; + } + result += type_name((*f_iter)->get_type(), false, true) + " " + + (name_params ? (*f_iter)->get_name() : "/* " + (*f_iter)->get_name() + " */"); + } + return result; +} + +/** + * Converts the parse type to a C++ enum string for the given type. + * + * @param type Thrift Type + * @return String of C++ code to definition of that type constant + */ +string t_cpp_generator::type_to_enum(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + return "apache::thrift::protocol::T_STRING"; + case t_base_type::TYPE_BOOL: + return "apache::thrift::protocol::T_BOOL"; + case t_base_type::TYPE_BYTE: + return "apache::thrift::protocol::T_BYTE"; + case t_base_type::TYPE_I16: + return "apache::thrift::protocol::T_I16"; + case t_base_type::TYPE_I32: + return "apache::thrift::protocol::T_I32"; + case t_base_type::TYPE_I64: + return "apache::thrift::protocol::T_I64"; + case t_base_type::TYPE_DOUBLE: + return "apache::thrift::protocol::T_DOUBLE"; + } + } else if (type->is_enum()) { + return "apache::thrift::protocol::T_I32"; + } else if (type->is_struct()) { + return "apache::thrift::protocol::T_STRUCT"; + } else if (type->is_xception()) { + return "apache::thrift::protocol::T_STRUCT"; + } else if (type->is_map()) { + return "apache::thrift::protocol::T_MAP"; + } else if (type->is_set()) { + return "apache::thrift::protocol::T_SET"; + } else if (type->is_list()) { + return "apache::thrift::protocol::T_LIST"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + +/** + * Returns the symbol name of the local reflection of a type. + */ +string t_cpp_generator::local_reflection_name(const char* prefix, t_type* ttype, bool external) { + ttype = get_true_type(ttype); + + // We have to use the program name as part of the identifier because + // if two thrift "programs" are compiled into one actual program + // you would get a symbol collison if they both defined list. + // trlo = Thrift Reflection LOcal. + string prog; + string name; + string nspace; + + // TODO(dreiss): Would it be better to pregenerate the base types + // and put them in Thrift.{h,cpp} ? + + if (ttype->is_base_type()) { + prog = program_->get_name(); + name = ttype->get_ascii_fingerprint(); + } else if (ttype->is_enum()) { + assert(ttype->get_program() != NULL); + prog = ttype->get_program()->get_name(); + name = ttype->get_ascii_fingerprint(); + } else if (ttype->is_container()) { + prog = program_->get_name(); + name = ttype->get_ascii_fingerprint(); + } else { + assert(ttype->is_struct() || ttype->is_xception()); + assert(ttype->get_program() != NULL); + prog = ttype->get_program()->get_name(); + name = ttype->get_ascii_fingerprint(); + } + + if (external && + ttype->get_program() != NULL && + ttype->get_program() != program_) { + nspace = namespace_prefix(ttype->get_program()->get_namespace("cpp")); + } + + return nspace + "trlo_" + prefix + "_" + prog + "_" + name; +} + +string t_cpp_generator::get_include_prefix(const t_program& program) const { + string include_prefix = program.get_include_prefix(); + if (!use_include_prefix_ || + (include_prefix.size() > 0 && include_prefix[0] == '/')) { + // if flag is turned off or this is absolute path, return empty prefix + return ""; + } + + string::size_type last_slash = string::npos; + if ((last_slash = include_prefix.rfind("/")) != string::npos) { + return include_prefix.substr(0, last_slash) + "/" + out_dir_base_ + "/"; + } + + return ""; +} + + +THRIFT_REGISTER_GENERATOR(cpp, "C++", +" dense: Generate type specifications for the dense protocol.\n" +" include_prefix: Use full include paths in generated files.\n" +); diff --git a/compiler/cpp/src/generate/t_csharp_generator.cc b/compiler/cpp/src/generate/t_csharp_generator.cc new file mode 100644 index 00000000..5a910eab --- /dev/null +++ b/compiler/cpp/src/generate/t_csharp_generator.cc @@ -0,0 +1,1700 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include "platform.h" +#include "t_oop_generator.h" +using namespace std; + + +class t_csharp_generator : public t_oop_generator +{ + public: + t_csharp_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_oop_generator(program) + { + out_dir_base_ = "gen-csharp"; + } + void init_generator(); + void close_generator(); + + void generate_consts(std::vector consts); + + void generate_typedef (t_typedef* ttypedef); + void generate_enum (t_enum* tenum); + void generate_struct (t_struct* tstruct); + void generate_xception (t_struct* txception); + void generate_service (t_service* tservice); + void generate_property(ofstream& out, t_field* tfield, bool isPublic); + bool print_const_value (std::ofstream& out, std::string name, t_type* type, t_const_value* value, bool in_static, bool defval=false, bool needtype=false); + std::string render_const_value(std::ofstream& out, std::string name, t_type* type, t_const_value* value); + void print_const_constructor(std::ofstream& out, std::vector consts); + void print_const_def_value(std::ofstream& out, std::string name, t_type* type, t_const_value* value); + + void generate_csharp_struct(t_struct* tstruct, bool is_exception); + void generate_csharp_struct_definition(std::ofstream& out, t_struct* tstruct, bool is_xception=false, bool in_class=false, bool is_result=false); + void generate_csharp_struct_reader(std::ofstream& out, t_struct* tstruct); + void generate_csharp_struct_result_writer(std::ofstream& out, t_struct* tstruct); + void generate_csharp_struct_writer(std::ofstream& out, t_struct* tstruct); + void generate_csharp_struct_tostring(std::ofstream& out, t_struct* tstruct); + + void generate_function_helpers(t_function* tfunction); + void generate_service_interface (t_service* tservice); + void generate_service_helpers (t_service* tservice); + void generate_service_client (t_service* tservice); + void generate_service_server (t_service* tservice); + void generate_process_function (t_service* tservice, t_function* function); + + void generate_deserialize_field (std::ofstream& out, t_field* tfield, std::string prefix=""); + void generate_deserialize_struct (std::ofstream& out, t_struct* tstruct, std::string prefix=""); + void generate_deserialize_container (std::ofstream& out, t_type* ttype, std::string prefix=""); + void generate_deserialize_set_element (std::ofstream& out, t_set* tset, std::string prefix=""); + void generate_deserialize_map_element (std::ofstream& out, t_map* tmap, std::string prefix=""); + void generate_deserialize_list_element (std::ofstream& out, t_list* list, std::string prefix=""); + void generate_serialize_field (std::ofstream& out, t_field* tfield, std::string prefix=""); + void generate_serialize_struct (std::ofstream& out, t_struct* tstruct, std::string prefix=""); + void generate_serialize_container (std::ofstream& out, t_type* ttype, std::string prefix=""); + void generate_serialize_map_element (std::ofstream& out, t_map* tmap, std::string iter, std::string map); + void generate_serialize_set_element (std::ofstream& out, t_set* tmap, std::string iter); + void generate_serialize_list_element (std::ofstream& out, t_list* tlist, std::string iter); + + void start_csharp_namespace (std::ofstream& out); + void end_csharp_namespace (std::ofstream& out); + + std::string csharp_type_usings(); + std::string csharp_thrift_usings(); + + std::string type_name(t_type* ttype, bool in_countainer=false, bool in_init=false); + std::string base_type_name(t_base_type* tbase, bool in_container=false); + std::string declare_field(t_field* tfield, bool init=false); + std::string function_signature(t_function* tfunction, std::string prefix=""); + std::string argument_list(t_struct* tstruct); + std::string type_to_enum(t_type* ttype); + std::string prop_name(t_field* tfield); + + bool type_can_be_null(t_type* ttype) { + while (ttype->is_typedef()) { + ttype = ((t_typedef*)ttype)->get_type(); + } + + return ttype->is_container() || + ttype->is_struct() || + ttype->is_xception() || + ttype->is_string(); + } + + private: + std::string namespace_name_; + std::ofstream f_service_; + std::string namespace_dir_; +}; + + +void t_csharp_generator::init_generator() { + MKDIR(get_out_dir().c_str()); + namespace_name_ = program_->get_namespace("csharp"); + + string dir = namespace_name_; + string subdir = get_out_dir().c_str(); + string::size_type loc; + + while ((loc = dir.find(".")) != string::npos) { + subdir = subdir + "/" + dir.substr(0, loc); + MKDIR(subdir.c_str()); + dir = dir.substr(loc + 1); + } + if (dir.size() > 0) { + subdir = subdir + "/" + dir; + MKDIR(subdir.c_str()); + } + + namespace_dir_ = subdir; +} + +void t_csharp_generator::start_csharp_namespace(ofstream& out) { + if (!namespace_name_.empty()) { + out << + "namespace " << namespace_name_ << "\n"; + scope_up(out); + } +} + +void t_csharp_generator::end_csharp_namespace(ofstream& out) { + if (!namespace_name_.empty()) { + scope_down(out); + } +} + +string t_csharp_generator::csharp_type_usings() { + return string() + + "using System;\n" + + "using System.Collections;\n" + + "using System.Collections.Generic;\n" + + "using System.Text;\n" + + "using System.IO;\n" + + "using Thrift;\n" + + "using Thrift.Collections;\n"; +} + +string t_csharp_generator::csharp_thrift_usings() { + return string() + + "using Thrift.Protocol;\n" + + "using Thrift.Transport;\n"; +} + +void t_csharp_generator::close_generator() { } +void t_csharp_generator::generate_typedef(t_typedef* ttypedef) {} + +void t_csharp_generator::generate_enum(t_enum* tenum) { + string f_enum_name = namespace_dir_+"/" + (tenum->get_name())+".cs"; + ofstream f_enum; + f_enum.open(f_enum_name.c_str()); + + f_enum << + autogen_comment() << endl; + + start_csharp_namespace(f_enum); + + indent(f_enum) << + "public enum " << tenum->get_name() << "\n"; + scope_up(f_enum); + + vector constants = tenum->get_constants(); + vector::iterator c_iter; + int value = -1; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) + { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + + indent(f_enum) << + (*c_iter)->get_name() << + " = " << value << "," << endl; + } + + scope_down(f_enum); + + end_csharp_namespace(f_enum); + + f_enum.close(); +} + +void t_csharp_generator::generate_consts(std::vector consts) { + if (consts.empty()){ + return; + } + string f_consts_name = namespace_dir_ + "/Constants.cs"; + ofstream f_consts; + f_consts.open(f_consts_name.c_str()); + + f_consts << + autogen_comment() << + csharp_type_usings() << endl; + + start_csharp_namespace(f_consts); + + indent(f_consts) << + "public class Constants" << endl; + scope_up(f_consts); + + vector::iterator c_iter; + bool need_static_constructor = false; + for (c_iter = consts.begin(); c_iter != consts.end(); ++c_iter) { + if (print_const_value(f_consts, (*c_iter)->get_name(), (*c_iter)->get_type(), (*c_iter)->get_value(), false)) { + need_static_constructor = true; + } + } + + if (need_static_constructor) { + print_const_constructor(f_consts, consts); + } + + scope_down(f_consts); + end_csharp_namespace(f_consts); + f_consts.close(); +} + +void t_csharp_generator::print_const_def_value(std::ofstream& out, string name, t_type* type, t_const_value* value) +{ + if (type->is_struct() || type->is_xception()) { + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + string val = render_const_value(out, name, field_type, v_iter->second); + indent(out) << name << "." << v_iter->first->get_string() << " = " << val << ";" << endl; + } + } else if (type->is_map()) { + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + string key = render_const_value(out, name, ktype, v_iter->first); + string val = render_const_value(out, name, vtype, v_iter->second); + indent(out) << name << "[" << key << "]" << " = " << val << ";" << endl; + } + } else if (type->is_list() || type->is_set()) { + t_type* etype; + if (type->is_list()) { + etype = ((t_list*)type)->get_elem_type(); + } else { + etype = ((t_set*)type)->get_elem_type(); + } + + const vector& val = value->get_list(); + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + string val = render_const_value(out, name, etype, *v_iter); + indent(out) << name << ".Add(" << val << ");" << endl; + } + } +} + +void t_csharp_generator::print_const_constructor(std::ofstream& out, std::vector consts) { + indent(out) << "static Constants()" << endl; + scope_up(out); + vector::iterator c_iter; + for (c_iter = consts.begin(); c_iter != consts.end(); ++c_iter) { + string name = (*c_iter)->get_name(); + t_type* type = (*c_iter)->get_type(); + t_const_value* value = (*c_iter)->get_value(); + + print_const_def_value(out, name, type, value); + } + scope_down(out); +} + + +//it seems like all that methods that call this are using in_static to be the opposite of what it would imply +bool t_csharp_generator::print_const_value(std::ofstream& out, string name, t_type* type, t_const_value* value, bool in_static, bool defval, bool needtype) { + indent(out); + bool need_static_construction = !in_static; + if (!defval || needtype) { + out << + (in_static ? "" : "public static ") << + type_name(type) << " "; + } + if (type->is_base_type()) { + string v2 = render_const_value(out, name, type, value); + out << name << " = " << v2 << ";" << endl; + need_static_construction = false; + } else if (type->is_enum()) { + out << name << " = (" << type_name(type, false, true) << ")" << value->get_integer() << ";" << endl; + need_static_construction = false; + } else if (type->is_struct() || type->is_xception()) { + out << name << " = new " << type_name(type) << "();" << endl; + } else if (type->is_map()) { + out << name << " = new " << type_name(type, true, true) << "();" << endl; + } else if (type->is_list() || type->is_set()) { + out << name << " = new " << type_name(type) << "();" << endl; + } + + if (defval && !type->is_base_type() && !type->is_enum()) { + print_const_def_value(out, name, type, value); + } + + return need_static_construction; +} + +std::string t_csharp_generator::render_const_value(ofstream& out, string name, t_type* type, t_const_value* value) { + std::ostringstream render; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + render << '"' << get_escaped_string(value) << '"'; + break; + case t_base_type::TYPE_BOOL: + render << ((value->get_integer() > 0) ? "true" : "false"); + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + render << value->get_integer(); + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() == t_const_value::CV_INTEGER) { + render << value->get_integer(); + } else { + render << value->get_double(); + } + break; + default: + throw "compiler error: no const of base type " + tbase; + } + } else if (type->is_enum()) { + render << "(" << type->get_name() << ")" << value->get_integer(); + } else { + string t = tmp("tmp"); + print_const_value(out, t, type, value, true, true, true); + render << t; + } + + return render.str(); +} + +void t_csharp_generator::generate_struct(t_struct* tstruct) { + generate_csharp_struct(tstruct, false); +} + +void t_csharp_generator::generate_xception(t_struct* txception) { + generate_csharp_struct(txception, true); +} + +void t_csharp_generator::generate_csharp_struct(t_struct* tstruct, bool is_exception) { + string f_struct_name = namespace_dir_ + "/" + (tstruct->get_name()) + ".cs"; + ofstream f_struct; + + f_struct.open(f_struct_name.c_str()); + + f_struct << + autogen_comment() << + csharp_type_usings() << + csharp_thrift_usings(); + + generate_csharp_struct_definition(f_struct, tstruct, is_exception); + + f_struct.close(); +} + +void t_csharp_generator::generate_csharp_struct_definition(ofstream &out, t_struct* tstruct, bool is_exception, bool in_class, bool is_result) { + + if (!in_class) { + start_csharp_namespace(out); + } + + out << endl; + indent(out) << "[Serializable]" << endl; + bool is_final = (tstruct->annotations_.find("final") != tstruct->annotations_.end()); + + indent(out) << "public " << (is_final ? "sealed " : "") << "class " << tstruct->get_name() << " : "; + + if (is_exception) { + out << "Exception, "; + } + out << "TBase"; + + out << endl; + + scope_up(out); + + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + + //make private members with public Properties + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + indent(out) << + "private " << declare_field(*m_iter, false) << endl; + } + out << endl; + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + generate_property(out, *m_iter, true); + } + + if (members.size() > 0) { + out << + endl << + indent() << "public Isset __isset;" << endl << + indent() << "[Serializable]" << endl << + indent() << "public struct Isset {" << endl; + indent_up(); + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + indent(out) << + "public bool " << (*m_iter)->get_name() << ";" << endl; + } + + indent_down(); + indent(out) << "}" << endl << endl; + } + + indent(out) << + "public " << tstruct->get_name() << "() {" << endl; + indent_up(); + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + t_type* t = (*m_iter)->get_type(); + while (t->is_typedef()) { + t = ((t_typedef*)t)->get_type(); + } + if ((*m_iter)->get_value() != NULL) { + print_const_value(out, "this." + (*m_iter)->get_name(), t, (*m_iter)->get_value(), true, true); + } + } + + indent_down(); + indent(out) << "}" << endl << endl; + + generate_csharp_struct_reader(out, tstruct); + if (is_result) { + generate_csharp_struct_result_writer(out, tstruct); + } else { + generate_csharp_struct_writer(out, tstruct); + } + generate_csharp_struct_tostring(out, tstruct); + scope_down(out); + out << endl; + + if (!in_class) + { + end_csharp_namespace(out); + } +} + +void t_csharp_generator::generate_csharp_struct_reader(ofstream& out, t_struct* tstruct) { + indent(out) << + "public void Read (TProtocol iprot)" << endl; + scope_up(out); + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + indent(out) << + "TField field;" << endl << + indent() << "iprot.ReadStructBegin();" << endl; + + indent(out) << + "while (true)" << endl; + scope_up(out); + + indent(out) << + "field = iprot.ReadFieldBegin();" << endl; + + indent(out) << + "if (field.Type == TType.Stop) { " << endl; + indent_up(); + indent(out) << + "break;" << endl; + indent_down(); + indent(out) << + "}" << endl; + + indent(out) << + "switch (field.ID)" << endl; + + scope_up(out); + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + indent(out) << + "case " << (*f_iter)->get_key() << ":" << endl; + indent_up(); + indent(out) << + "if (field.Type == " << type_to_enum((*f_iter)->get_type()) << ") {" << endl; + indent_up(); + + generate_deserialize_field(out, *f_iter, "this."); + indent(out) << + "this.__isset." << (*f_iter)->get_name() << " = true;" << endl; + indent_down(); + out << + indent() << "} else { " << endl << + indent() << " TProtocolUtil.Skip(iprot, field.Type);" << endl << + indent() << "}" << endl << + indent() << "break;" << endl; + indent_down(); + } + + indent(out) << + "default: " << endl; + indent_up(); + indent(out) << "TProtocolUtil.Skip(iprot, field.Type);" << endl; + indent(out) << "break;" << endl; + indent_down(); + + scope_down(out); + + indent(out) << + "iprot.ReadFieldEnd();" << endl; + + scope_down(out); + + indent(out) << + "iprot.ReadStructEnd();" << endl; + + indent_down(); + + indent(out) << "}" << endl << endl; + +} + +void t_csharp_generator::generate_csharp_struct_writer(ofstream& out, t_struct* tstruct) { + out << + indent() << "public void Write(TProtocol oprot) {" << endl; + indent_up(); + + string name = tstruct->get_name(); + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + indent(out) << + "TStruct struc = new TStruct(\"" << name << "\");" << endl; + indent(out) << + "oprot.WriteStructBegin(struc);" << endl; + + if (fields.size() > 0) { + indent(out) << "TField field = new TField();" << endl; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + bool null_allowed = type_can_be_null((*f_iter)->get_type()); + if (null_allowed) { + indent(out) << + "if (this." << (*f_iter)->get_name() << " != null && __isset." << (*f_iter)->get_name() << ") {" << endl; + indent_up(); + } + else + { + indent(out) << + "if (__isset." << (*f_iter)->get_name() << ") {" << endl; + indent_up(); + } + + indent(out) << + "field.Name = \"" << (*f_iter)->get_name() << "\";" << endl; + indent(out) << + "field.Type = " << type_to_enum((*f_iter)->get_type()) << ";" << endl; + indent(out) << + "field.ID = " << (*f_iter)->get_key() << ";" << endl; + indent(out) << + "oprot.WriteFieldBegin(field);" << endl; + + generate_serialize_field(out, *f_iter, "this."); + + indent(out) << + "oprot.WriteFieldEnd();" << endl; + + indent_down(); + indent(out) << "}" << endl; + } + } + + indent(out) << + "oprot.WriteFieldStop();" << endl; + indent(out) << + "oprot.WriteStructEnd();" << endl; + + indent_down(); + + indent(out) << + "}" << endl << endl; +} + +void t_csharp_generator::generate_csharp_struct_result_writer(ofstream& out, t_struct* tstruct) { + indent(out) << + "public void Write(TProtocol oprot) {" << endl; + indent_up(); + + string name = tstruct->get_name(); + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + indent(out) << + "TStruct struc = new TStruct(\"" << name << "\");" << endl; + indent(out) << + "oprot.WriteStructBegin(struc);" << endl; + + if (fields.size() > 0) { + indent(out) << "TField field = new TField();" << endl; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + out << + endl << indent() << "if "; + } else { + out << + " else if "; + } + + out << + "(this.__isset." << (*f_iter)->get_name() << ") {" << endl; + indent_up(); + + bool null_allowed = type_can_be_null((*f_iter)->get_type()); + if (null_allowed) { + indent(out) << + "if (this." << (*f_iter)->get_name() << " != null) {" << endl; + indent_up(); + } + + indent(out) << + "field.Name = \"" << (*f_iter)->get_name() << "\";" << endl; + indent(out) << + "field.Type = " << type_to_enum((*f_iter)->get_type()) << ";" << endl; + indent(out) << + "field.ID = " << (*f_iter)->get_key() << ";" << endl; + indent(out) << + "oprot.WriteFieldBegin(field);" << endl; + + generate_serialize_field(out, *f_iter, "this."); + + indent(out) << + "oprot.WriteFieldEnd();" << endl; + + if (null_allowed) { + indent_down(); + indent(out) << "}" << endl; + } + + indent_down(); + indent(out) << "}"; + } + } + + out << + endl << + indent() << "oprot.WriteFieldStop();" << endl << + indent() << "oprot.WriteStructEnd();" << endl; + + indent_down(); + + indent(out) << + "}" << endl << endl; +} + +void t_csharp_generator::generate_csharp_struct_tostring(ofstream& out, t_struct* tstruct) { + indent(out) << + "public override string ToString() {" << endl; + indent_up(); + + indent(out) << + "StringBuilder sb = new StringBuilder(\"" << tstruct->get_name() << "(\");" << endl; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + bool first = true; + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + indent(out) << + "sb.Append(\"" << (*f_iter)->get_name() << ": \");" << endl; + } else { + indent(out) << + "sb.Append(\"," << (*f_iter)->get_name() << ": \");" << endl; + } + t_type* ttype = (*f_iter)->get_type(); + if (ttype->is_xception() || ttype->is_struct()) { + indent(out) << + "sb.Append(this." << (*f_iter)->get_name() << "== null ? \"\" : "<< "this." << (*f_iter)->get_name() << ".ToString());" << endl; + } else { + indent(out) << + "sb.Append(this." << (*f_iter)->get_name() << ");" << endl; + } + } + + indent(out) << + "sb.Append(\")\");" << endl; + indent(out) << + "return sb.ToString();" << endl; + + indent_down(); + indent(out) << "}" << endl << endl; +} + +void t_csharp_generator::generate_service(t_service* tservice) { + string f_service_name = namespace_dir_ + "/" + service_name_ + ".cs"; + f_service_.open(f_service_name.c_str()); + + f_service_ << + autogen_comment() << + csharp_type_usings() << + csharp_thrift_usings(); + + start_csharp_namespace(f_service_); + + indent(f_service_) << + "public class " << service_name_ << " {" << endl; + indent_up(); + + generate_service_interface(tservice); + generate_service_client(tservice); + generate_service_server(tservice); + generate_service_helpers(tservice); + + indent_down(); + + indent(f_service_) << + "}" << endl; + end_csharp_namespace(f_service_); + f_service_.close(); +} + +void t_csharp_generator::generate_service_interface(t_service* tservice) { + string extends = ""; + string extends_iface = ""; + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + extends_iface = " : " + extends + ".Iface"; + } + + indent(f_service_) << + "public interface Iface" << extends_iface << " {" << endl; + indent_up(); + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) + { + indent(f_service_) << + function_signature(*f_iter) << ";" << endl; + } + indent_down(); + f_service_ << + indent() << "}" << endl << endl; +} + +void t_csharp_generator::generate_service_helpers(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* ts = (*f_iter)->get_arglist(); + generate_csharp_struct_definition(f_service_, ts, false, true); + generate_function_helpers(*f_iter); + } +} + +void t_csharp_generator::generate_service_client(t_service* tservice) { + string extends = ""; + string extends_client = ""; + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + extends_client = extends + ".Client, "; + } + + indent(f_service_) << + "public class Client : " << extends_client << "Iface {" << endl; + indent_up(); + indent(f_service_) << + "public Client(TProtocol prot) : this(prot, prot)" << endl; + scope_up(f_service_); + scope_down(f_service_); + f_service_ << endl; + + indent(f_service_) << + "public Client(TProtocol iprot, TProtocol oprot)"; + if (!extends.empty()) { + f_service_ << " : base(iprot, oprot)"; + } + f_service_ << endl; + + scope_up(f_service_); + if (extends.empty()) { + f_service_ << + indent() << "iprot_ = iprot;" << endl << + indent() << "oprot_ = oprot;" << endl; + } + scope_down(f_service_); + + f_service_ << endl; + + if (extends.empty()) { + f_service_ << + indent() << "protected TProtocol iprot_;" << endl << + indent() << "protected TProtocol oprot_;" << endl << + indent() << "protected int seqid_;" << endl << endl; + + f_service_ << indent() << "public TProtocol InputProtocol" << endl; + scope_up(f_service_); + indent(f_service_) << "get { return iprot_; }" << endl; + scope_down(f_service_); + + f_service_ << indent() << "public TProtocol OutputProtocol" << endl; + scope_up(f_service_); + indent(f_service_) << "get { return oprot_; }" << endl; + scope_down(f_service_); + f_service_ << endl << endl; + } + + vector functions = tservice->get_functions(); + vector::const_iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string funname = (*f_iter)->get_name(); + + indent(f_service_) << + "public " << function_signature(*f_iter) << endl; + scope_up(f_service_); + indent(f_service_) << + "send_" << funname << "("; + + t_struct* arg_struct = (*f_iter)->get_arglist(); + + const vector& fields = arg_struct->get_members(); + vector::const_iterator fld_iter; + bool first = true; + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << (*fld_iter)->get_name(); + } + f_service_ << ");" << endl; + + if (!(*f_iter)->is_oneway()) { + f_service_ << indent(); + if (!(*f_iter)->get_returntype()->is_void()) { + f_service_ << "return "; + } + f_service_ << + "recv_" << funname << "();" << endl; + } + scope_down(f_service_); + f_service_ << endl; + + t_function send_function(g_type_void, + string("send_") + (*f_iter)->get_name(), + (*f_iter)->get_arglist()); + + string argsname = (*f_iter)->get_name() + "_args"; + + indent(f_service_) << + "public " << function_signature(&send_function) << endl; + scope_up(f_service_); + + f_service_ << + indent() << "oprot_.WriteMessageBegin(new TMessage(\"" << funname << "\", TMessageType.Call, seqid_));" << endl << + indent() << argsname << " args = new " << argsname << "();" << endl; + + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + f_service_ << + indent() << "args." << prop_name(*fld_iter) << " = " << (*fld_iter)->get_name() << ";" << endl; + } + + f_service_ << + indent() << "args.Write(oprot_);" << endl << + indent() << "oprot_.WriteMessageEnd();" << endl << + indent() << "oprot_.Transport.Flush();" << endl; + + scope_down(f_service_); + f_service_ << endl; + + if (!(*f_iter)->is_oneway()) { + string resultname = (*f_iter)->get_name() + "_result"; + + t_struct noargs(program_); + t_function recv_function((*f_iter)->get_returntype(), + string("recv_") + (*f_iter)->get_name(), + &noargs, + (*f_iter)->get_xceptions()); + indent(f_service_) << + "public " << function_signature(&recv_function) << endl; + scope_up(f_service_); + + f_service_ << + indent() << "TMessage msg = iprot_.ReadMessageBegin();" << endl << + indent() << "if (msg.Type == TMessageType.Exception) {" << endl; + indent_up(); + f_service_ << + indent() << "TApplicationException x = TApplicationException.Read(iprot_);" << endl << + indent() << "iprot_.ReadMessageEnd();" << endl << + indent() << "throw x;" << endl; + indent_down(); + f_service_ << + indent() << "}" << endl << + indent() << resultname << " result = new " << resultname << "();" << endl << + indent() << "result.Read(iprot_);" << endl << + indent() << "iprot_.ReadMessageEnd();" << endl; + + if (!(*f_iter)->get_returntype()->is_void()) { + f_service_ << + indent() << "if (result.__isset.success) {" << endl << + indent() << " return result.Success;" << endl << + indent() << "}" << endl; + } + + t_struct *xs = (*f_iter)->get_xceptions(); + + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << + indent() << "if (result.__isset." << (*x_iter)->get_name() << ") {" << endl << + indent() << " throw result." << prop_name(*x_iter) << ";" << endl << + indent() << "}" << endl; + } + + if ((*f_iter)->get_returntype()->is_void()) { + indent(f_service_) << + "return;" << endl; + } else { + f_service_ << + indent() << "throw new TApplicationException(TApplicationException.ExceptionType.MissingResult, \"" << (*f_iter)->get_name() << " failed: unknown result\");" << endl; + } + + scope_down(f_service_); + f_service_ << endl; + } + } + + indent_down(); + indent(f_service_) << + "}" << endl; +} + +void t_csharp_generator::generate_service_server(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + string extends = ""; + string extends_processor = ""; + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + extends_processor = extends + ".Processor, "; + } + + indent(f_service_) << + "public class Processor : " << extends_processor << "TProcessor {" << endl; + indent_up(); + + indent(f_service_) << + "public Processor(Iface iface)" ; + if (!extends.empty()) { + f_service_ << " : base(iface)"; + } + f_service_ << endl; + scope_up(f_service_); + f_service_ << + indent() << "iface_ = iface;" << endl; + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_service_ << + indent() << "processMap_[\"" << (*f_iter)->get_name() << "\"] = " << (*f_iter)->get_name() << "_Process;" << endl; + } + + scope_down(f_service_); + f_service_ << endl; + + if (extends.empty()) { + f_service_ << + indent() << "protected delegate void ProcessFunction(int seqid, TProtocol iprot, TProtocol oprot);" << endl; + } + + f_service_ << + indent() << "private Iface iface_;" << endl; + + if (extends.empty()) { + f_service_ << + indent() << "protected Dictionary processMap_ = new Dictionary();" << endl; + } + + f_service_ << endl; + + if (extends.empty()) { + indent(f_service_) << + "public bool Process(TProtocol iprot, TProtocol oprot)" << endl; + } + else + { + indent(f_service_) << + "public new bool Process(TProtocol iprot, TProtocol oprot)" << endl; + } + scope_up(f_service_); + + f_service_ << indent() << "try" << endl; + scope_up(f_service_); + + f_service_ << + indent() << "TMessage msg = iprot.ReadMessageBegin();" << endl; + + f_service_ << + indent() << "ProcessFunction fn;" << endl << + indent() << "processMap_.TryGetValue(msg.Name, out fn);" << endl << + indent() << "if (fn == null) {" << endl << + indent() << " TProtocolUtil.Skip(iprot, TType.Struct);" << endl << + indent() << " iprot.ReadMessageEnd();" << endl << + indent() << " TApplicationException x = new TApplicationException (TApplicationException.ExceptionType.UnknownMethod, \"Invalid method name: '\" + msg.Name + \"'\");" << endl << + indent() << " oprot.WriteMessageBegin(new TMessage(msg.Name, TMessageType.Exception, msg.SeqID));" << endl << + indent() << " x.Write(oprot);" << endl << + indent() << " oprot.WriteMessageEnd();" << endl << + indent() << " oprot.Transport.Flush();" << endl << + indent() << " return true;" << endl << + indent() << "}" << endl << + indent() << "fn(msg.SeqID, iprot, oprot);" << endl; + + scope_down(f_service_); + + f_service_ << + indent() << "catch (IOException)" << endl; + scope_up(f_service_); + f_service_ << + indent() << "return false;" << endl; + scope_down(f_service_); + + f_service_ << + indent() << "return true;" << endl; + + scope_down(f_service_); + f_service_ << endl; + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) + { + generate_process_function(tservice, *f_iter); + } + + indent_down(); + indent(f_service_) << + "}" << endl << endl; +} + +void t_csharp_generator::generate_function_helpers(t_function* tfunction) { + if (tfunction->is_oneway()) { + return; + } + + t_struct result(program_, tfunction->get_name() + "_result"); + t_field success(tfunction->get_returntype(), "success", 0); + if (!tfunction->get_returntype()->is_void()) { + result.append(&success); + } + + t_struct *xs = tfunction->get_xceptions(); + const vector& fields = xs->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + result.append(*f_iter); + } + + generate_csharp_struct_definition(f_service_, &result, false, true, true); +} + +void t_csharp_generator::generate_process_function(t_service* tservice, t_function* tfunction) { + indent(f_service_) << + "public void " << tfunction->get_name() << "_Process(int seqid, TProtocol iprot, TProtocol oprot)" << endl; + scope_up(f_service_); + + string argsname = tfunction->get_name() + "_args"; + string resultname = tfunction->get_name() + "_result"; + + f_service_ << + indent() << argsname << " args = new " << argsname << "();" << endl << + indent() << "args.Read(iprot);" << endl << + indent() << "iprot.ReadMessageEnd();" << endl; + + t_struct* xs = tfunction->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + + if (!tfunction->is_oneway()) { + f_service_ << + indent() << resultname << " result = new " << resultname << "();" << endl; + } + + if (xceptions.size() > 0) { + f_service_ << + indent() << "try {" << endl; + indent_up(); + } + + t_struct* arg_struct = tfunction->get_arglist(); + const std::vector& fields = arg_struct->get_members(); + vector::const_iterator f_iter; + + f_service_ << indent(); + if (!tfunction->is_oneway() && !tfunction->get_returntype()->is_void()) { + f_service_ << "result.Success = "; + } + f_service_ << + "iface_." << tfunction->get_name() << "("; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << "args." << prop_name(*f_iter); + } + f_service_ << ");" << endl; + + if (!tfunction->is_oneway() && xceptions.size() > 0) { + indent_down(); + f_service_ << indent() << "}"; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << " catch (" << type_name((*x_iter)->get_type(), false, false) << " " << (*x_iter)->get_name() << ") {" << endl; + if (!tfunction->is_oneway()) { + indent_up(); + f_service_ << + indent() << "result." << prop_name(*x_iter) << " = " << (*x_iter)->get_name() << ";" << endl; + indent_down(); + f_service_ << indent() << "}"; + } else { + f_service_ << "}"; + } + } + f_service_ << endl; + } + + if (tfunction->is_oneway()) { + f_service_ << + indent() << "return;" << endl; + scope_down(f_service_); + + return; + } + + f_service_ << + indent() << "oprot.WriteMessageBegin(new TMessage(\"" << tfunction->get_name() << "\", TMessageType.Reply, seqid)); " << endl << + indent() << "result.Write(oprot);" << endl << + indent() << "oprot.WriteMessageEnd();" << endl << + indent() << "oprot.Transport.Flush();" << endl; + + scope_down(f_service_); + + f_service_ << endl; +} + +void t_csharp_generator::generate_deserialize_field(ofstream& out, t_field* tfield, string prefix) { + t_type* type = tfield->get_type(); + while(type->is_typedef()) { + type = ((t_typedef*)type)->get_type(); + } + + if (type->is_void()) { + throw "CANNOT GENERATE DESERIALIZE CODE FOR void TYPE: " + prefix + tfield->get_name(); + } + + string name = prefix + tfield->get_name(); + + if (type->is_struct() || type->is_xception()) { + generate_deserialize_struct(out, (t_struct*)type, name); + } else if (type->is_container()) { + generate_deserialize_container(out, type, name); + } else if (type->is_base_type() || type->is_enum()) { + indent(out) << + name << " = "; + + if (type->is_enum()) + { + out << "(" << type_name(type, false, true) << ")"; + } + + out << "iprot."; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "compiler error: cannot serialize void field in a struct: " + name; + break; + case t_base_type::TYPE_STRING: + if (((t_base_type*)type)->is_binary()) { + out << "ReadBinary();"; + } else { + out << "ReadString();"; + } + break; + case t_base_type::TYPE_BOOL: + out << "ReadBool();"; + break; + case t_base_type::TYPE_BYTE: + out << "ReadByte();"; + break; + case t_base_type::TYPE_I16: + out << "ReadI16();"; + break; + case t_base_type::TYPE_I32: + out << "ReadI32();"; + break; + case t_base_type::TYPE_I64: + out << "ReadI64();"; + break; + case t_base_type::TYPE_DOUBLE: + out << "ReadDouble();"; + break; + default: + throw "compiler error: no C# name for base type " + tbase; + } + } else if (type->is_enum()) { + out << "ReadI32();"; + } + out << endl; + } else { + printf("DO NOT KNOW HOW TO DESERIALIZE FIELD '%s' TYPE '%s'\n", tfield->get_name().c_str(), type_name(type).c_str()); + } +} + +void t_csharp_generator::generate_deserialize_struct(ofstream& out, t_struct* tstruct, string prefix) { + out << + indent() << prefix << " = new " << type_name(tstruct) << "();" << endl << + indent() << prefix << ".Read(iprot);" << endl; +} + +void t_csharp_generator::generate_deserialize_container(ofstream& out, t_type* ttype, string prefix) { + scope_up(out); + + string obj; + + if (ttype->is_map()) { + obj = tmp("_map"); + } else if (ttype->is_set()) { + obj = tmp("_set"); + } else if (ttype->is_list()) { + obj = tmp("_list"); + } + + indent(out) << + prefix << " = new " << type_name(ttype, false, true) << "();" <is_map()) { + out << + indent() << "TMap " << obj << " = iprot.ReadMapBegin();" << endl; + } else if (ttype->is_set()) { + out << + indent() << "TSet " << obj << " = iprot.ReadSetBegin();" << endl; + } else if (ttype->is_list()) { + out << + indent() << "TList " << obj << " = iprot.ReadListBegin();" << endl; + } + + string i = tmp("_i"); + indent(out) << + "for( int " << i << " = 0; " << i << " < " << obj << ".Count" << "; " << "++" << i << ")" << endl; + scope_up(out); + + if (ttype->is_map()) { + generate_deserialize_map_element(out, (t_map*)ttype, prefix); + } else if (ttype->is_set()) { + generate_deserialize_set_element(out, (t_set*)ttype, prefix); + } else if (ttype->is_list()) { + generate_deserialize_list_element(out, (t_list*)ttype, prefix); + } + + scope_down(out); + + if (ttype->is_map()) { + indent(out) << "iprot.ReadMapEnd();" << endl; + } else if (ttype->is_set()) { + indent(out) << "iprot.ReadSetEnd();" << endl; + } else if (ttype->is_list()) { + indent(out) << "iprot.ReadListEnd();" << endl; + } + + scope_down(out); +} + +void t_csharp_generator::generate_deserialize_map_element(ofstream& out, t_map* tmap, string prefix) { + string key = tmp("_key"); + string val = tmp("_val"); + + t_field fkey(tmap->get_key_type(), key); + t_field fval(tmap->get_val_type(), val); + + indent(out) << + declare_field(&fkey) << endl; + indent(out) << + declare_field(&fval) << endl; + + generate_deserialize_field(out, &fkey); + generate_deserialize_field(out, &fval); + + indent(out) << + prefix << "[" << key << "] = " << val << ";" << endl; +} + +void t_csharp_generator::generate_deserialize_set_element(ofstream& out, t_set* tset, string prefix) { + string elem = tmp("_elem"); + t_field felem(tset->get_elem_type(), elem); + + indent(out) << + declare_field(&felem, true) << endl; + + generate_deserialize_field(out, &felem); + + indent(out) << + prefix << ".Add(" << elem << ");" << endl; +} + +void t_csharp_generator::generate_deserialize_list_element(ofstream& out, t_list* tlist, string prefix) { + string elem = tmp("_elem"); + t_field felem(tlist->get_elem_type(), elem); + + indent(out) << + declare_field(&felem, true) << endl; + + generate_deserialize_field(out, &felem); + + indent(out) << + prefix << ".Add(" << elem << ");" << endl; +} + +void t_csharp_generator::generate_serialize_field(ofstream& out, t_field* tfield, string prefix) { + t_type* type = tfield->get_type(); + while (type->is_typedef()) { + type = ((t_typedef*)type)->get_type(); + } + + if (type->is_void()) { + throw "CANNOT GENERATE SERIALIZE CODE FOR void TYPE: " + prefix + tfield->get_name(); + } + + if (type->is_struct() || type->is_xception()) { + generate_serialize_struct(out, (t_struct*)type, prefix + tfield->get_name()); + } else if (type->is_container()) { + generate_serialize_container(out, type, prefix + tfield->get_name()); + } else if (type->is_base_type() || type->is_enum()) { + string name = prefix + tfield->get_name(); + indent(out) << + "oprot."; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + + switch(tbase) { + case t_base_type::TYPE_VOID: + throw "compiler error: cannot serialize void field in a struct: " + name; + break; + case t_base_type::TYPE_STRING: + if (((t_base_type*)type)->is_binary()) { + out << "WriteBinary("; + } else { + out << "WriteString("; + } + out << name << ");"; + break; + case t_base_type::TYPE_BOOL: + out << "WriteBool(" << name << ");"; + break; + case t_base_type::TYPE_BYTE: + out << "WriteByte(" << name << ");"; + break; + case t_base_type::TYPE_I16: + out << "WriteI16(" << name << ");"; + break; + case t_base_type::TYPE_I32: + out << "WriteI32(" << name << ");"; + break; + case t_base_type::TYPE_I64: + out << "WriteI64(" << name << ");"; + break; + case t_base_type::TYPE_DOUBLE: + out << "WriteDouble(" << name << ");"; + break; + default: + throw "compiler error: no C# name for base type " + tbase; + } + } else if (type->is_enum()) { + out << "WriteI32((int)" << name << ");"; + } + out << endl; + } else { + printf("DO NOT KNOW HOW TO SERIALIZE '%s%s' TYPE '%s'\n", + prefix.c_str(), + tfield->get_name().c_str(), + type_name(type).c_str()); + } +} + +void t_csharp_generator::generate_serialize_struct(ofstream& out, t_struct* tstruct, string prefix) { + out << + indent() << prefix << ".Write(oprot);" << endl; +} + +void t_csharp_generator::generate_serialize_container(ofstream& out, t_type* ttype, string prefix) { + scope_up(out); + + if (ttype->is_map()) { + indent(out) << + "oprot.WriteMapBegin(new TMap(" << + type_to_enum(((t_map*)ttype)->get_key_type()) << ", " << + type_to_enum(((t_map*)ttype)->get_val_type()) << ", " << + prefix << ".Count));" << endl; + } else if (ttype->is_set()) { + indent(out) << + "oprot.WriteSetBegin(new TSet(" << + type_to_enum(((t_set*)ttype)->get_elem_type()) << ", " << + prefix << ".Count));" << endl; + } else if (ttype->is_list()) { + indent(out) << + "oprot.WriteListBegin(new TList(" << + type_to_enum(((t_list*)ttype)->get_elem_type()) << ", " << + prefix << ".Count));" << endl; + } + + string iter = tmp("_iter"); + if (ttype->is_map()) { + indent(out) << + "foreach (" << + type_name(((t_map*)ttype)->get_key_type()) << " " << iter << + " in " << + prefix << ".Keys)"; + } else if (ttype->is_set()) { + indent(out) << + "foreach (" << + type_name(((t_set*)ttype)->get_elem_type()) << " " << iter << + " in " << + prefix << ")"; + } else if (ttype->is_list()) { + indent(out) << + "foreach (" << + type_name(((t_list*)ttype)->get_elem_type()) << " " << iter << + " in " << + prefix << ")"; + } + + out << endl; + scope_up(out); + + if (ttype->is_map()) { + generate_serialize_map_element(out, (t_map*)ttype, iter, prefix); + } else if (ttype->is_set()) { + generate_serialize_set_element(out, (t_set*)ttype, iter); + } else if (ttype->is_list()) { + generate_serialize_list_element(out, (t_list*)ttype, iter); + } + + if (ttype->is_map()) { + indent(out) << "oprot.WriteMapEnd();" << endl; + } else if (ttype->is_set()) { + indent(out) << "oprot.WriteSetEnd();" << endl; + } else if (ttype->is_list()) { + indent(out) << "oprot.WriteListEnd();" << endl; + } + + scope_down(out); + scope_down(out); +} + +void t_csharp_generator::generate_serialize_map_element(ofstream& out, t_map* tmap, string iter, string map) { + t_field kfield(tmap->get_key_type(), iter); + generate_serialize_field(out, &kfield, ""); + t_field vfield(tmap->get_val_type(), map + "[" + iter + "]"); + generate_serialize_field(out, &vfield, ""); +} + +void t_csharp_generator::generate_serialize_set_element(ofstream& out, t_set* tset, string iter) { + t_field efield(tset->get_elem_type(), iter); + generate_serialize_field(out, &efield, ""); +} + +void t_csharp_generator::generate_serialize_list_element(ofstream& out, t_list* tlist, string iter) { + t_field efield(tlist->get_elem_type(), iter); + generate_serialize_field(out, &efield, ""); +} + +void t_csharp_generator::generate_property(ofstream& out, t_field* tfield, bool isPublic) { + indent(out) << (isPublic ? "public " : "private ") << type_name(tfield->get_type()) + << " " << prop_name(tfield) << endl; + scope_up(out); + indent(out) << "get" << endl; + scope_up(out); + indent(out) << "return " << tfield->get_name() << ";" << endl; + scope_down(out); + indent(out) << "set" << endl; + scope_up(out); + indent(out) << "__isset." << tfield->get_name() << " = true;" << endl; + indent(out) << "this." << tfield->get_name() << " = value;" << endl; + scope_down(out); + scope_down(out); + out << endl; +} + +std::string t_csharp_generator::prop_name(t_field* tfield) { + string name (tfield->get_name()); + name[0] = toupper(name[0]); + return name; +} + +string t_csharp_generator::type_name(t_type* ttype, bool in_container, bool in_init) { + while (ttype->is_typedef()) { + ttype = ((t_typedef*)ttype)->get_type(); + } + + if (ttype->is_base_type()) { + return base_type_name((t_base_type*)ttype, in_container); + } else if (ttype->is_map()) { + t_map *tmap = (t_map*) ttype; + return "Dictionary<" + type_name(tmap->get_key_type(), true) + + ", " + type_name(tmap->get_val_type(), true) + ">"; + } else if (ttype->is_set()) { + t_set* tset = (t_set*) ttype; + return "THashSet<" + type_name(tset->get_elem_type(), true) + ">"; + } else if (ttype->is_list()) { + t_list* tlist = (t_list*) ttype; + return "List<" + type_name(tlist->get_elem_type(), true) + ">"; + } + + t_program* program = ttype->get_program(); + if (program != NULL && program != program_) { + string ns = program->get_namespace("csharp"); + if (!ns.empty()) { + return ns + "." + ttype->get_name(); + } + } + + return ttype->get_name(); +} + +string t_csharp_generator::base_type_name(t_base_type* tbase, bool in_container) { + switch (tbase->get_base()) { + case t_base_type::TYPE_VOID: + return "void"; + case t_base_type::TYPE_STRING: + if (tbase->is_binary()) { + return "byte[]"; + } else { + return "string"; + } + case t_base_type::TYPE_BOOL: + return "bool"; + case t_base_type::TYPE_BYTE: + return "byte"; + case t_base_type::TYPE_I16: + return "short"; + case t_base_type::TYPE_I32: + return "int"; + case t_base_type::TYPE_I64: + return "long"; + case t_base_type::TYPE_DOUBLE: + return "double"; + default: + throw "compiler error: no C# name for base type " + tbase->get_base(); + } +} + +string t_csharp_generator::declare_field(t_field* tfield, bool init) { + string result = type_name(tfield->get_type()) + " " + tfield->get_name(); + if (init) { + t_type* ttype = tfield->get_type(); + while (ttype->is_typedef()) { + ttype = ((t_typedef*)ttype)->get_type(); + } + if (ttype->is_base_type() && tfield->get_value() != NULL) { + ofstream dummy; + result += " = " + render_const_value(dummy, tfield->get_name(), ttype, tfield->get_value()); + } else if (ttype->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)ttype)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + result += " = null"; + break; + case t_base_type::TYPE_BOOL: + result += " = false"; + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + result += " = 0"; + break; + case t_base_type::TYPE_DOUBLE: + result += " = (double)0"; + break; + } + } else if (ttype->is_enum()) { + result += " = (" + type_name(ttype, false, true) + ")0"; + } else if (ttype->is_container()) { + result += " = new " + type_name(ttype, false, true) + "()"; + } else { + result += " = new " + type_name(ttype, false, true) + "()"; + } + } + return result + ";"; +} + +string t_csharp_generator::function_signature(t_function* tfunction, string prefix) { + t_type* ttype = tfunction->get_returntype(); + return type_name(ttype) + " " + prefix + tfunction->get_name() + "(" + argument_list(tfunction->get_arglist()) + ")"; +} + +string t_csharp_generator::argument_list(t_struct* tstruct) { + string result = ""; + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + result += ", "; + } + result += type_name((*f_iter)->get_type()) + " " + (*f_iter)->get_name(); + } + return result; +} + +string t_csharp_generator::type_to_enum(t_type* type) { + while (type->is_typedef()) { + type = ((t_typedef*)type)->get_type(); + } + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + return "TType.String"; + case t_base_type::TYPE_BOOL: + return "TType.Bool"; + case t_base_type::TYPE_BYTE: + return "TType.Byte"; + case t_base_type::TYPE_I16: + return "TType.I16"; + case t_base_type::TYPE_I32: + return "TType.I32"; + case t_base_type::TYPE_I64: + return "TType.I64"; + case t_base_type::TYPE_DOUBLE: + return "TType.Double"; + } + } else if (type->is_enum()) { + return "TType.I32"; + } else if (type->is_struct() || type->is_xception()) { + return "TType.Struct"; + } else if (type->is_map()) { + return "TType.Map"; + } else if (type->is_set()) { + return "TType.Set"; + } else if (type->is_list()) { + return "TType.List"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + + +THRIFT_REGISTER_GENERATOR(csharp, "C#", ""); diff --git a/compiler/cpp/src/generate/t_erl_generator.cc b/compiler/cpp/src/generate/t_erl_generator.cc new file mode 100644 index 00000000..0aff4f39 --- /dev/null +++ b/compiler/cpp/src/generate/t_erl_generator.cc @@ -0,0 +1,932 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include "t_generator.h" +#include "platform.h" + +using namespace std; + + +/** + * Erlang code generator. + * + */ +class t_erl_generator : public t_generator { + public: + t_erl_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_generator(program) + { + program_name_[0] = tolower(program_name_[0]); + service_name_[0] = tolower(service_name_[0]); + out_dir_base_ = "gen-erl"; + } + + /** + * Init and close methods + */ + + void init_generator(); + void close_generator(); + + /** + * Program-level generation functions + */ + + void generate_typedef (t_typedef* ttypedef); + void generate_enum (t_enum* tenum); + void generate_const (t_const* tconst); + void generate_struct (t_struct* tstruct); + void generate_xception (t_struct* txception); + void generate_service (t_service* tservice); + + std::string render_const_value(t_type* type, t_const_value* value); + + /** + * Struct generation code + */ + + void generate_erl_struct(t_struct* tstruct, bool is_exception); + void generate_erl_struct_definition(std::ostream& out, std::ostream& hrl_out, t_struct* tstruct, bool is_xception=false, bool is_result=false); + void generate_erl_struct_info(std::ostream& out, t_struct* tstruct); + void generate_erl_function_helpers(t_function* tfunction); + + /** + * Service-level generation functions + */ + + void generate_service_helpers (t_service* tservice); + void generate_service_interface (t_service* tservice); + void generate_function_info (t_service* tservice, t_function* tfunction); + + /** + * Helper rendering functions + */ + + std::string erl_autogen_comment(); + std::string erl_imports(); + std::string render_includes(); + std::string declare_field(t_field* tfield); + std::string type_name(t_type* ttype); + + std::string function_signature(t_function* tfunction, std::string prefix=""); + + + std::string argument_list(t_struct* tstruct); + std::string type_to_enum(t_type* ttype); + std::string generate_type_term(t_type* ttype, bool expand_structs); + std::string type_module(t_type* ttype); + + std::string capitalize(std::string in) { + in[0] = toupper(in[0]); + return in; + } + + std::string uncapitalize(std::string in) { + in[0] = tolower(in[0]); + return in; + } + + private: + + /** + * add function to export list + */ + + void export_function(t_function* tfunction, std::string prefix=""); + void export_string(std::string name, int num); + + void export_types_function(t_function* tfunction, std::string prefix=""); + void export_types_string(std::string name, int num); + + /** + * write out headers and footers for hrl files + */ + + void hrl_header(std::ostream& out, std::string name); + void hrl_footer(std::ostream& out, std::string name); + + /** + * stuff to spit out at the top of generated files + */ + + bool export_lines_first_; + std::ostringstream export_lines_; + + bool export_types_lines_first_; + std::ostringstream export_types_lines_; + + /** + * File streams + */ + + std::ostringstream f_types_; + std::ofstream f_types_file_; + std::ofstream f_types_hrl_file_; + + std::ofstream f_consts_; + std::ostringstream f_service_; + std::ofstream f_service_file_; + std::ofstream f_service_hrl_; + +}; + + +/** + * UI for file generation by opening up the necessary file output + * streams. + * + * @param tprogram The program to generate + */ +void t_erl_generator::init_generator() { + // Make output directory + MKDIR(get_out_dir().c_str()); + + // setup export lines + export_lines_first_ = true; + export_types_lines_first_ = true; + + // types files + string f_types_name = get_out_dir()+program_name_+"_types.erl"; + string f_types_hrl_name = get_out_dir()+program_name_+"_types.hrl"; + + f_types_file_.open(f_types_name.c_str()); + f_types_hrl_file_.open(f_types_hrl_name.c_str()); + + hrl_header(f_types_hrl_file_, program_name_ + "_types"); + + f_types_file_ << + erl_autogen_comment() << endl << + "-module(" << program_name_ << "_types)." << endl << + erl_imports() << endl; + + f_types_file_ << + "-include(\"" << program_name_ << "_types.hrl\")." << endl << + endl; + + f_types_hrl_file_ << render_includes() << endl; + + // consts file + string f_consts_name = get_out_dir()+program_name_+"_constants.hrl"; + f_consts_.open(f_consts_name.c_str()); + + f_consts_ << + erl_autogen_comment() << endl << + erl_imports() << endl << + "-include(\"" << program_name_ << "_types.hrl\")." << endl << + endl; +} + +/** + * Boilerplate at beginning and end of header files + */ +void t_erl_generator::hrl_header(ostream& out, string name) { + out << "-ifndef(_" << name << "_included)." << endl << + "-define(_" << name << "_included, yeah)." << endl; +} + +void t_erl_generator::hrl_footer(ostream& out, string name) { + out << "-endif." << endl; +} + +/** + * Renders all the imports necessary for including another Thrift program + */ +string t_erl_generator::render_includes() { + const vector& includes = program_->get_includes(); + string result = ""; + for (size_t i = 0; i < includes.size(); ++i) { + result += "-include(\"" + includes[i]->get_name() + "_types.hrl\").\n"; + } + if (includes.size() > 0) { + result += "\n"; + } + return result; +} + +/** + * Autogen'd comment + */ +string t_erl_generator::erl_autogen_comment() { + return + std::string("%%\n") + + "%% Autogenerated by Thrift\n" + + "%%\n" + + "%% DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n" + + "%%\n"; +} + +/** + * Prints standard thrift imports + */ +string t_erl_generator::erl_imports() { + return ""; +} + +/** + * Closes the type files + */ +void t_erl_generator::close_generator() { + // Close types file + export_types_string("struct_info", 1); + + f_types_file_ << "-export([" << export_types_lines_.str() << "])." << endl; + f_types_file_ << f_types_.str(); + f_types_file_ << "struct_info('i am a dummy struct') -> undefined." << endl; + + hrl_footer(f_types_hrl_file_, string("BOGUS")); + + f_types_file_.close(); + f_types_hrl_file_.close(); + f_consts_.close(); +} + +/** + * Generates a typedef. no op + * + * @param ttypedef The type definition + */ +void t_erl_generator::generate_typedef(t_typedef* ttypedef) { +} + +/** + * Generates code for an enumerated type. Done using a class to scope + * the values. + * + * @param tenum The enumeration + */ +void t_erl_generator::generate_enum(t_enum* tenum) { + vector constants = tenum->get_constants(); + vector::iterator c_iter; + + int value = -1; + + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + + string name = capitalize((*c_iter)->get_name()); + + f_types_hrl_file_ << + indent() << "-define(" << program_name_ << "_" << name << ", " << value << ")."<< endl; + } + + f_types_hrl_file_ << endl; +} + +/** + * Generate a constant value + */ +void t_erl_generator::generate_const(t_const* tconst) { + t_type* type = tconst->get_type(); + string name = capitalize(tconst->get_name()); + t_const_value* value = tconst->get_value(); + + f_consts_ << "-define(" << program_name_ << "_" << name << ", " << render_const_value(type, value) << ")." << endl << endl; +} + +/** + * Prints the value of a constant with the given type. Note that type checking + * is NOT performed in this function as it is always run beforehand using the + * validate_types method in main.cc + */ +string t_erl_generator::render_const_value(t_type* type, t_const_value* value) { + type = get_true_type(type); + std::ostringstream out; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + out << '"' << get_escaped_string(value) << '"'; + break; + case t_base_type::TYPE_BOOL: + out << (value->get_integer() > 0 ? "true" : "false"); + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + out << value->get_integer(); + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() == t_const_value::CV_INTEGER) { + out << value->get_integer(); + } else { + out << value->get_double(); + } + break; + default: + throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + indent(out) << value->get_integer(); + + } else if (type->is_struct() || type->is_xception()) { + out << "#" << type->get_name() << "{"; + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + const map& val = value->get_map(); + map::const_iterator v_iter; + + bool first = true; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + + if (first) { + first = false; + } else { + out << ","; + } + out << v_iter->first->get_string(); + out << " = "; + out << render_const_value(field_type, v_iter->second); + } + indent_down(); + indent(out) << "}"; + + } else if (type->is_map()) { + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + const map& val = value->get_map(); + map::const_iterator v_iter; + + bool first = true; + out << "dict:from_list(["; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + if (first) { + first=false; + } else { + out << ","; + } + out << "(" + << render_const_value(ktype, v_iter->first) << "," + << render_const_value(vtype, v_iter->second) << ")"; + } + out << "])"; + + } else if (type->is_set()) { + t_type* etype; + etype = ((t_set*)type)->get_elem_type(); + + bool first = true; + const vector& val = value->get_list(); + vector::const_iterator v_iter; + out << "sets:from_list(["; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + if (first) { + first=false; + } else { + out << ","; + } + out << "(" << render_const_value(etype, *v_iter) << ",true)"; + } + out << "])"; + + } else if (type->is_list()) { + t_type* etype; + etype = ((t_list*)type)->get_elem_type(); + out << "["; + + bool first = true; + const vector& val = value->get_list(); + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + if (first) { + first=false; + } else { + out << ","; + } + out << render_const_value(etype, *v_iter); + } + out << "]"; + } else { + throw "CANNOT GENERATE CONSTANT FOR TYPE: " + type->get_name(); + } + return out.str(); +} + +/** + * Generates a struct + */ +void t_erl_generator::generate_struct(t_struct* tstruct) { + generate_erl_struct(tstruct, false); +} + +/** + * Generates a struct definition for a thrift exception. Basically the same + * as a struct but extends the Exception class. + * + * @param txception The struct definition + */ +void t_erl_generator::generate_xception(t_struct* txception) { + generate_erl_struct(txception, true); +} + +/** + * Generates a struct + */ +void t_erl_generator::generate_erl_struct(t_struct* tstruct, + bool is_exception) { + generate_erl_struct_definition(f_types_, f_types_hrl_file_, tstruct, is_exception); +} + +/** + * Generates a struct definition for a thrift data type. + * + * @param tstruct The struct definition + */ +void t_erl_generator::generate_erl_struct_definition(ostream& out, + ostream& hrl_out, + t_struct* tstruct, + bool is_exception, + bool is_result) +{ + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + + indent(out) << "%% struct " << type_name(tstruct) << endl; + + if (is_exception) { + } + + out << endl; + + if (members.size() > 0) { + indent(out) << "% -record(" << type_name(tstruct) << ", {"; + indent(hrl_out) << "-record(" << type_name(tstruct) << ", {"; + + bool first = true; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + if (first) { + first = false; + } else { + out << ", "; + hrl_out << ", "; + } + std::string name = uncapitalize((*m_iter)->get_name()); + out << name; + hrl_out << name; + } + out << "})." << endl; + hrl_out << "})." << endl; + } else { // no members; explicit comment + indent(out) << "% -record(" << type_name(tstruct) << ", {})." << endl; + indent(hrl_out) << "-record(" << type_name(tstruct) << ", {})." << endl; + } + + out << endl; + hrl_out << endl; + + + generate_erl_struct_info(out, tstruct); +} + +/** + * Generates the read method for a struct + */ +void t_erl_generator::generate_erl_struct_info(ostream& out, + t_struct* tstruct) { + string name = type_name(tstruct); + + indent(out) << "struct_info('" << name << "') ->" << endl; + indent_up(); + + out << indent() << generate_type_term(tstruct, true) << ";" << endl; + + indent_down(); + out << endl; +} + + +/** + * Generates a thrift service. + * + * @param tservice The service definition + */ +void t_erl_generator::generate_service(t_service* tservice) { + // somehow this point is reached before the constructor and it's not downcased yet + // ...awesome + service_name_[0] = tolower(service_name_[0]); + + string f_service_hrl_name = get_out_dir()+service_name_+"_thrift.hrl"; + string f_service_name = get_out_dir()+service_name_+"_thrift.erl"; + f_service_file_.open(f_service_name.c_str()); + f_service_hrl_.open(f_service_hrl_name.c_str()); + + // Reset service text aggregating stream streams + f_service_.str(""); + export_lines_.str(""); + export_lines_first_ = true; + + hrl_header(f_service_hrl_, service_name_); + + if (tservice->get_extends() != NULL) { + f_service_hrl_ << "-include(\"" << + uncapitalize(tservice->get_extends()->get_name()) << "_thrift.hrl\"). % inherit " << endl; + } + + f_service_hrl_ << + "-include(\"" << program_name_ << "_types.hrl\")." << endl << + endl; + + // Generate the three main parts of the service (well, two for now in PHP) + generate_service_helpers(tservice); // cpiro: New Erlang Order + + generate_service_interface(tservice); + + // indent_down(); + + f_service_file_ << + erl_autogen_comment() << endl << + "-module(" << service_name_ << "_thrift)." << endl << + "-behaviour(thrift_service)." << endl << endl << + erl_imports() << endl; + + f_service_file_ << "-include(\"" << uncapitalize(tservice->get_name()) << "_thrift.hrl\")." << endl << endl; + + f_service_file_ << "-export([" << export_lines_.str() << "])." << endl << endl; + + f_service_file_ << f_service_.str(); + + hrl_footer(f_service_hrl_, f_service_name); + + // Close service file + f_service_file_.close(); + f_service_hrl_.close(); +} + +/** + * Generates helper functions for a service. + * + * @param tservice The service to generate a header definition for + */ +void t_erl_generator::generate_service_helpers(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + // indent(f_service_) << + // "% HELPER FUNCTIONS AND STRUCTURES" << endl << endl; + + export_string("struct_info", 1); + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + generate_erl_function_helpers(*f_iter); + } + f_service_ << "struct_info('i am a dummy struct') -> undefined." << endl; +} + +/** + * Generates a struct and helpers for a function. + * + * @param tfunction The function + */ +void t_erl_generator::generate_erl_function_helpers(t_function* tfunction) { +} + +/** + * Generates a service interface definition. + * + * @param tservice The service to generate a header definition for + */ +void t_erl_generator::generate_service_interface(t_service* tservice) { + + export_string("function_info", 2); + + vector functions = tservice->get_functions(); + vector::iterator f_iter; + f_service_ << "%%% interface" << endl; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_service_ << + indent() << "% " << function_signature(*f_iter) << endl; + + generate_function_info(tservice, *f_iter); + } + + // Inheritance - pass unknown functions to base class + if (tservice->get_extends() != NULL) { + indent(f_service_) << "function_info(Function, InfoType) ->" << endl; + indent_up(); + indent(f_service_) << uncapitalize(tservice->get_extends()->get_name()) + << "_thrift:function_info(Function, InfoType)." << endl; + indent_down(); + } else { + // Dummy function_info so we don't worry about the ;s + indent(f_service_) << "function_info(xxx, dummy) -> dummy." << endl; + } + + indent(f_service_) << endl; +} + + +/** + * Generates a function_info(FunctionName, params_type) and + * function_info(FunctionName, reply_type) + */ +void t_erl_generator::generate_function_info(t_service* tservice, + t_function* tfunction) { + + string name_atom = "'" + tfunction->get_name() + "'"; + + + + t_struct* xs = tfunction->get_xceptions(); + t_struct* arg_struct = tfunction->get_arglist(); + + // function_info(Function, params_type): + indent(f_service_) << + "function_info(" << name_atom << ", params_type) ->" << endl; + indent_up(); + + indent(f_service_) << generate_type_term(arg_struct, true) << ";" << endl; + + indent_down(); + + // function_info(Function, reply_type): + indent(f_service_) << + "function_info(" << name_atom << ", reply_type) ->" << endl; + indent_up(); + + if (!tfunction->get_returntype()->is_void()) + indent(f_service_) << + generate_type_term(tfunction->get_returntype(), false) << ";" << endl; + else if (tfunction->is_oneway()) + indent(f_service_) << "oneway_void;" << endl; + else + indent(f_service_) << "{struct, []}" << ";" << endl; + indent_down(); + + // function_info(Function, exceptions): + indent(f_service_) << + "function_info(" << name_atom << ", exceptions) ->" << endl; + indent_up(); + indent(f_service_) << generate_type_term(xs, true) << ";" << endl; + indent_down(); +} + + +/** + * Declares a field, which may include initialization as necessary. + * + * @param ttype The type + */ +string t_erl_generator::declare_field(t_field* tfield) { // TODO + string result = "@" + tfield->get_name(); + t_type* type = get_true_type(tfield->get_type()); + if (tfield->get_value() != NULL) { + result += " = " + render_const_value(type, tfield->get_value()); + } else { + result += " = nil"; + } + return result; +} + +/** + * Renders a function signature of the form 'type name(args)' + * + * @param tfunction Function definition + * @return String of rendered function definition + */ +string t_erl_generator::function_signature(t_function* tfunction, + string prefix) { + return + prefix + tfunction->get_name() + + "(This" + capitalize(argument_list(tfunction->get_arglist())) + ")"; +} + +/** + * Add a function to the exports list + */ +void t_erl_generator::export_string(string name, int num) { + if (export_lines_first_) { + export_lines_first_ = false; + } else { + export_lines_ << ", "; + } + export_lines_ << name << "/" << num; +} + +void t_erl_generator::export_types_function(t_function* tfunction, + string prefix) { + + export_types_string(prefix + tfunction->get_name(), + 1 // This + + ((tfunction->get_arglist())->get_members()).size() + ); +} + +void t_erl_generator::export_types_string(string name, int num) { + if (export_types_lines_first_) { + export_types_lines_first_ = false; + } else { + export_types_lines_ << ", "; + } + export_types_lines_ << name << "/" << num; +} + +void t_erl_generator::export_function(t_function* tfunction, + string prefix) { + + export_string(prefix + tfunction->get_name(), + 1 // This + + ((tfunction->get_arglist())->get_members()).size() + ); +} + + +/** + * Renders a field list + */ +string t_erl_generator::argument_list(t_struct* tstruct) { + string result = ""; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + result += ", "; // initial comma to compensate for initial This + } else { + result += ", "; + } + result += capitalize((*f_iter)->get_name()); + } + return result; +} + +string t_erl_generator::type_name(t_type* ttype) { + string prefix = ""; + string name = ttype->get_name(); + + if (ttype->is_struct() || ttype->is_xception() || ttype->is_service()) { + name = uncapitalize(ttype->get_name()); + } + + return prefix + name; +} + +/** + * Converts the parse type to a Erlang "type" (macro for int constants) + */ +string t_erl_generator::type_to_enum(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + return "?tType_STRING"; + case t_base_type::TYPE_BOOL: + return "?tType_BOOL"; + case t_base_type::TYPE_BYTE: + return "?tType_BYTE"; + case t_base_type::TYPE_I16: + return "?tType_I16"; + case t_base_type::TYPE_I32: + return "?tType_I32"; + case t_base_type::TYPE_I64: + return "?tType_I64"; + case t_base_type::TYPE_DOUBLE: + return "?tType_DOUBLE"; + } + } else if (type->is_enum()) { + return "?tType_I32"; + } else if (type->is_struct() || type->is_xception()) { + return "?tType_STRUCT"; + } else if (type->is_map()) { + return "?tType_MAP"; + } else if (type->is_set()) { + return "?tType_SET"; + } else if (type->is_list()) { + return "?tType_LIST"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + + +/** + * Generate an Erlang term which represents a thrift type + */ +std::string t_erl_generator::generate_type_term(t_type* type, + bool expand_structs) { + type = get_true_type(type); + + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + return "string"; + case t_base_type::TYPE_BOOL: + return "bool"; + case t_base_type::TYPE_BYTE: + return "byte"; + case t_base_type::TYPE_I16: + return "i16"; + case t_base_type::TYPE_I32: + return "i32"; + case t_base_type::TYPE_I64: + return "i64"; + case t_base_type::TYPE_DOUBLE: + return "double"; + } + } else if (type->is_enum()) { + return "i32"; + } else if (type->is_struct() || type->is_xception()) { + if (expand_structs) { + // Convert to format: {struct, [{Fid, TypeTerm}, {Fid, TypeTerm}...]} + std::stringstream ret; + + + ret << "{struct, ["; + + int first = true; + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + // Comma separate the tuples + if (!first) ret << "," << endl << indent(); + first = false; + + ret << "{" << (*f_iter)->get_key() << ", " << + generate_type_term((*f_iter)->get_type(), false) << "}"; + } + + ret << "]}" << endl; + + return ret.str(); + } else { + return "{struct, {'" + type_module(type) + "', '" + type_name(type) + "'}}"; + } + } else if (type->is_map()) { + // {map, KeyType, ValType} + t_type *key_type = ((t_map*)type)->get_key_type(); + t_type *val_type = ((t_map*)type)->get_val_type(); + + return "{map, " + generate_type_term(key_type, false) + ", " + + generate_type_term(val_type, false) + "}"; + + } else if (type->is_set()) { + t_type *elem_type = ((t_set*)type)->get_elem_type(); + + return "{set, " + generate_type_term(elem_type, false) + "}"; + + } else if (type->is_list()) { + t_type *elem_type = ((t_list*)type)->get_elem_type(); + + return "{list, " + generate_type_term(elem_type, false) + "}"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + +std::string t_erl_generator::type_module(t_type* ttype) { + return uncapitalize(ttype->get_program()->get_name()) + "_types"; +} + +THRIFT_REGISTER_GENERATOR(erl, "Erlang", ""); diff --git a/compiler/cpp/src/generate/t_generator.cc b/compiler/cpp/src/generate/t_generator.cc new file mode 100644 index 00000000..38c053c5 --- /dev/null +++ b/compiler/cpp/src/generate/t_generator.cc @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "t_generator.h" +using namespace std; + +/** + * Top level program generation function. Calls the generator subclass methods + * for preparing file streams etc. then iterates over all the parts of the + * program to perform the correct actions. + * + * @param program The thrift program to compile into C++ source + */ +void t_generator::generate_program() { + // Initialize the generator + init_generator(); + + // Generate enums + vector enums = program_->get_enums(); + vector::iterator en_iter; + for (en_iter = enums.begin(); en_iter != enums.end(); ++en_iter) { + generate_enum(*en_iter); + } + + // Generate typedefs + vector typedefs = program_->get_typedefs(); + vector::iterator td_iter; + for (td_iter = typedefs.begin(); td_iter != typedefs.end(); ++td_iter) { + generate_typedef(*td_iter); + } + + // Generate constants + vector consts = program_->get_consts(); + generate_consts(consts); + + // Generate structs and exceptions in declared order + vector objects = program_->get_objects(); + vector::iterator o_iter; + for (o_iter = objects.begin(); o_iter != objects.end(); ++o_iter) { + if ((*o_iter)->is_xception()) { + generate_xception(*o_iter); + } else { + generate_struct(*o_iter); + } + } + + // Generate services + vector services = program_->get_services(); + vector::iterator sv_iter; + for (sv_iter = services.begin(); sv_iter != services.end(); ++sv_iter) { + service_name_ = get_service_name(*sv_iter); + generate_service(*sv_iter); + } + + // Close the generator + close_generator(); +} + +string t_generator::escape_string(const string &in) const { + string result = ""; + for (string::const_iterator it = in.begin(); it < in.end(); it++) { + std::map::const_iterator res = escape_.find(*it); + if (res != escape_.end()) { + result.append(res->second); + } else { + result.push_back(*it); + } + } + return result; +} + +void t_generator::generate_consts(vector consts) { + vector::iterator c_iter; + for (c_iter = consts.begin(); c_iter != consts.end(); ++c_iter) { + generate_const(*c_iter); + } +} + +void t_generator::generate_docstring_comment(ofstream& out, + const string& comment_start, + const string& line_prefix, + const string& contents, + const string& comment_end) { + if (comment_start != "") indent(out) << comment_start; + stringstream docs(contents, ios_base::in); + while (!docs.eof()) { + char line[1024]; + docs.getline(line, 1024); + if (strlen(line) > 0 || !docs.eof()) { // skip the empty last line + indent(out) << line_prefix << line << std::endl; + } + } + if (comment_end != "") indent(out) << comment_end; +} + + +void t_generator_registry::register_generator(t_generator_factory* factory) { + gen_map_t& the_map = get_generator_map(); + if (the_map.find(factory->get_short_name()) != the_map.end()) { + failure("Duplicate generators for language \"%s\"!\n", factory->get_short_name().c_str()); + } + the_map[factory->get_short_name()] = factory; +} + +t_generator* t_generator_registry::get_generator(t_program* program, + const string& options) { + string::size_type colon = options.find(':'); + string language = options.substr(0, colon); + + map parsed_options; + if (colon != string::npos) { + string::size_type pos = colon+1; + while (pos != string::npos && pos < options.size()) { + string::size_type next_pos = options.find(',', pos); + string option = options.substr(pos, next_pos-pos); + pos = ((next_pos == string::npos) ? next_pos : next_pos+1); + + string::size_type separator = option.find('='); + string key, value; + if (separator == string::npos) { + key = option; + value = ""; + } else { + key = option.substr(0, separator); + value = option.substr(separator+1); + } + + parsed_options[key] = value; + } + } + + gen_map_t& the_map = get_generator_map(); + gen_map_t::iterator iter = the_map.find(language); + + if (iter == the_map.end()) { + return NULL; + } + + return iter->second->get_generator(program, parsed_options, options); +} + +t_generator_registry::gen_map_t& t_generator_registry::get_generator_map() { + // http://www.parashift.com/c++-faq-lite/ctors.html#faq-10.12 + static gen_map_t* the_map = new gen_map_t(); + return *the_map; +} + +t_generator_factory::t_generator_factory( + const std::string& short_name, + const std::string& long_name, + const std::string& documentation) + : short_name_(short_name) + , long_name_(long_name) + , documentation_(documentation) +{ + t_generator_registry::register_generator(this); +} diff --git a/compiler/cpp/src/generate/t_generator.h b/compiler/cpp/src/generate/t_generator.h new file mode 100644 index 00000000..7514fb16 --- /dev/null +++ b/compiler/cpp/src/generate/t_generator.h @@ -0,0 +1,321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_GENERATOR_H +#define T_GENERATOR_H + +#include +#include +#include +#include +#include "parse/t_program.h" +#include "globals.h" + +/** + * Base class for a thrift code generator. This class defines the basic + * routines for code generation and contains the top level method that + * dispatches code generation across various components. + * + */ +class t_generator { + public: + t_generator(t_program* program) { + tmp_ = 0; + indent_ = 0; + program_ = program; + program_name_ = get_program_name(program); + escape_['\n'] = "\\n"; + escape_['\r'] = "\\r"; + escape_['\t'] = "\\t"; + escape_['"'] = "\\\""; + escape_['\\'] = "\\\\"; + } + + virtual ~t_generator() {} + + /** + * Framework generator method that iterates over all the parts of a program + * and performs general actions. This is implemented by the base class and + * should not normally be overwritten in the subclasses. + */ + virtual void generate_program(); + + const t_program* get_program() const { return program_; } + + void generate_docstring_comment(std::ofstream& out, + const std::string& comment_start, + const std::string& line_prefix, + const std::string& contents, + const std::string& comment_end); + + /** + * Escape string to use one in generated sources. + */ + virtual std::string escape_string(const std::string &in) const; + + std::string get_escaped_string(t_const_value* constval) { + return escape_string(constval->get_string()); + } + + protected: + + /** + * Optional methods that may be imlemented by subclasses to take necessary + * steps at the beginning or end of code generation. + */ + + virtual void init_generator() {} + virtual void close_generator() {} + + virtual void generate_consts(std::vector consts); + + /** + * Pure virtual methods implemented by the generator subclasses. + */ + + virtual void generate_typedef (t_typedef* ttypedef) = 0; + virtual void generate_enum (t_enum* tenum) = 0; + virtual void generate_const (t_const* tconst) {} + virtual void generate_struct (t_struct* tstruct) = 0; + virtual void generate_service (t_service* tservice) = 0; + virtual void generate_xception (t_struct* txception) { + // By default exceptions are the same as structs + generate_struct(txception); + } + + /** + * Method to get the program name, may be overridden + */ + virtual std::string get_program_name(t_program* tprogram) { + return tprogram->get_name(); + } + + /** + * Method to get the service name, may be overridden + */ + virtual std::string get_service_name(t_service* tservice) { + return tservice->get_name(); + } + + /** + * Get the current output directory + */ + virtual std::string get_out_dir() const { + return program_->get_out_path() + out_dir_base_ + "/"; + } + + /** + * Creates a unique temporary variable name, which is just "name" with a + * number appended to it (i.e. name35) + */ + std::string tmp(std::string name) { + std::ostringstream out; + out << name << tmp_++; + return out.str(); + } + + /** + * Indentation level modifiers + */ + + void indent_up(){ + ++indent_; + } + + void indent_down() { + --indent_; + } + + /** + * Indentation print function + */ + std::string indent() { + std::string ind = ""; + int i; + for (i = 0; i < indent_; ++i) { + ind += " "; + } + return ind; + } + + /** + * Indentation utility wrapper + */ + std::ostream& indent(std::ostream &os) { + return os << indent(); + } + + /** + * Capitalization helpers + */ + std::string capitalize(std::string in) { + in[0] = toupper(in[0]); + return in; + } + std::string decapitalize(std::string in) { + in[0] = tolower(in[0]); + return in; + } + std::string lowercase(std::string in) { + for (size_t i = 0; i < in.size(); ++i) { + in[i] = tolower(in[i]); + } + return in; + } + std::string underscore(std::string in) { + in[0] = tolower(in[0]); + for (size_t i = 1; i < in.size(); ++i) { + if (isupper(in[i])) { + in[i] = tolower(in[i]); + in.insert(i, "_"); + } + } + return in; + } + + /** + * Get the true type behind a series of typedefs. + */ + static t_type* get_true_type(t_type* type) { + while (type->is_typedef()) { + type = ((t_typedef*)type)->get_type(); + } + return type; + } + + protected: + /** + * The program being generated + */ + t_program* program_; + + /** + * Quick accessor for formatted program name that is currently being + * generated. + */ + std::string program_name_; + + /** + * Quick accessor for formatted service name that is currently being + * generated. + */ + std::string service_name_; + + /** + * Output type-specifc directory name ("gen-*") + */ + std::string out_dir_base_; + + /** + * Map of characters to escape in string literals. + */ + std::map escape_; + + private: + /** + * Current code indentation level + */ + int indent_; + + /** + * Temporary variable counter, for making unique variable names + */ + int tmp_; +}; + + +/** + * A factory for producing generator classes of a particular language. + * + * This class is also responsible for: + * - Registering itself with the generator registry. + * - Providing documentation for the generators it produces. + */ +class t_generator_factory { + public: + t_generator_factory(const std::string& short_name, + const std::string& long_name, + const std::string& documentation); + + virtual ~t_generator_factory() {} + + virtual t_generator* get_generator( + // The program to generate. + t_program* program, + // Note: parsed_options will not exist beyond the call to get_generator. + const std::map& parsed_options, + // Note: option_string might not exist beyond the call to get_generator. + const std::string& option_string) + = 0; + + std::string get_short_name() { return short_name_; } + std::string get_long_name() { return long_name_; } + std::string get_documentation() { return documentation_; } + + private: + std::string short_name_; + std::string long_name_; + std::string documentation_; +}; + +template +class t_generator_factory_impl : public t_generator_factory { + public: + t_generator_factory_impl(const std::string& short_name, + const std::string& long_name, + const std::string& documentation) + : t_generator_factory(short_name, long_name, documentation) + {} + + virtual t_generator* get_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) { + return new generator(program, parsed_options, option_string); + } +}; + +class t_generator_registry { + public: + static void register_generator(t_generator_factory* factory); + + static t_generator* get_generator(t_program* program, + const std::string& options); + + typedef std::map gen_map_t; + static gen_map_t& get_generator_map(); + + private: + t_generator_registry(); + t_generator_registry(const t_generator_registry&); +}; + +#define THRIFT_REGISTER_GENERATOR(language, long_name, doc) \ + class t_##language##_generator_factory_impl \ + : public t_generator_factory_impl \ + { \ + public: \ + t_##language##_generator_factory_impl() \ + : t_generator_factory_impl( \ + #language, long_name, doc) \ + {} \ + }; \ + static t_##language##_generator_factory_impl _registerer; + +#endif diff --git a/compiler/cpp/src/generate/t_hs_generator.cc b/compiler/cpp/src/generate/t_hs_generator.cc new file mode 100644 index 00000000..c8fda774 --- /dev/null +++ b/compiler/cpp/src/generate/t_hs_generator.cc @@ -0,0 +1,1445 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include "t_oop_generator.h" +#include "platform.h" +using namespace std; + + +/** + * Haskell code generator. + * + */ +class t_hs_generator : public t_oop_generator { + public: + t_hs_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_oop_generator(program) + { + out_dir_base_ = "gen-hs"; + } + + /** + * Init and close methods + */ + + void init_generator(); + void close_generator(); + + /** + * Program-level generation functions + */ + void generate_typedef (t_typedef* ttypedef); + void generate_enum (t_enum* tenum); + void generate_const (t_const* tconst); + void generate_struct (t_struct* tstruct); + void generate_xception (t_struct* txception); + void generate_service (t_service* tservice); + + std::string render_const_value(t_type* type, t_const_value* value); + + /** + * Struct generation code + */ + + void generate_hs_struct(t_struct* tstruct, bool is_exception); + void generate_hs_struct_definition(std::ofstream &out,t_struct* tstruct, bool is_xception=false,bool helper=false); + void generate_hs_struct_reader(std::ofstream& out, t_struct* tstruct); + void generate_hs_struct_writer(std::ofstream& out, t_struct* tstruct); + void generate_hs_function_helpers(t_function* tfunction); + + /** + * Service-level generation functions + */ + + void generate_service_helpers (t_service* tservice); + void generate_service_interface (t_service* tservice); + void generate_service_client (t_service* tservice); + void generate_service_server (t_service* tservice); + void generate_process_function (t_service* tservice, t_function* tfunction); + + /** + * Serialization constructs + */ + + void generate_deserialize_field (std::ofstream &out, + t_field* tfield, + std::string prefix); + + void generate_deserialize_struct (std::ofstream &out, + t_struct* tstruct); + + void generate_deserialize_container (std::ofstream &out, + t_type* ttype); + + void generate_deserialize_set_element (std::ofstream &out, + t_set* tset); + + + void generate_deserialize_list_element (std::ofstream &out, + t_list* tlist, + std::string prefix=""); + void generate_deserialize_type (std::ofstream &out, + t_type* type); + + void generate_serialize_field (std::ofstream &out, + t_field* tfield, + std::string name= ""); + + void generate_serialize_struct (std::ofstream &out, + t_struct* tstruct, + std::string prefix=""); + + void generate_serialize_container (std::ofstream &out, + t_type* ttype, + std::string prefix=""); + + void generate_serialize_map_element (std::ofstream &out, + t_map* tmap, + std::string kiter, + std::string viter); + + void generate_serialize_set_element (std::ofstream &out, + t_set* tmap, + std::string iter); + + void generate_serialize_list_element (std::ofstream &out, + t_list* tlist, + std::string iter); + + /** + * Helper rendering functions + */ + + std::string hs_autogen_comment(); + std::string hs_imports(); + std::string type_name(t_type* ttype); + std::string function_type(t_function* tfunc, bool options = false, bool io = false, bool method = false); + std::string type_to_enum(t_type* ttype); + std::string render_hs_type(t_type* type, bool needs_parens = true); + + + private: + + /** + * File streams + */ + + std::ofstream f_types_; + std::ofstream f_consts_; + std::ofstream f_service_; + std::ofstream f_iface_; + std::ofstream f_client_; + +}; + + +/** + * Prepares for file generation by opening up the necessary file output + * streams. + * + * @param tprogram The program to generate + */ +void t_hs_generator::init_generator() { + // Make output directory + MKDIR(get_out_dir().c_str()); + + // Make output file + + string pname = capitalize(program_name_); + string f_types_name = get_out_dir()+pname+"_Types.hs"; + f_types_.open(f_types_name.c_str()); + + string f_consts_name = get_out_dir()+pname+"_Consts.hs"; + f_consts_.open(f_consts_name.c_str()); + + // Print header + f_types_ << + hs_autogen_comment() << endl << + "module " << pname <<"_Types where" << endl << + hs_imports() << endl; + + f_consts_ << + hs_autogen_comment() << endl << + "module " << pname <<"_Consts where" << endl << + hs_imports() << endl << + "import " << pname<<"_Types"<< endl; + +} + + +/** + * Autogen'd comment + */ +string t_hs_generator::hs_autogen_comment() { + return + std::string("-----------------------------------------------------------------\n") + + "-- Autogenerated by Thrift --\n" + + "-- --\n" + + "-- DO NOT EDIT UNLESS YOU ARE SURE YOU KNOW WHAT YOU ARE DOING --\n" + + "-----------------------------------------------------------------\n"; +} + +/** + * Prints standard thrift imports + */ +string t_hs_generator::hs_imports() { + return "import Thrift\nimport Data.Typeable ( Typeable )\nimport Control.Exception\nimport qualified Data.Map as Map\nimport qualified Data.Set as Set\nimport Data.Int"; +} + +/** + * Closes the type files + */ +void t_hs_generator::close_generator() { + // Close types file + f_types_.close(); + f_consts_.close(); +} + +/** + * Generates a typedef. Ez. + * + * @param ttypedef The type definition + */ +void t_hs_generator::generate_typedef(t_typedef* ttypedef) { + f_types_ << + indent() << "type "<< capitalize(ttypedef->get_symbolic()) << " = " << render_hs_type(ttypedef->get_type(), false) << endl << endl; +} + +/** + * Generates code for an enumerated type. + * the values. + * + * @param tenum The enumeration + */ +void t_hs_generator::generate_enum(t_enum* tenum) { + indent(f_types_) << "data "<get_name())<<" = "; + indent_up(); + vector constants = tenum->get_constants(); + vector::iterator c_iter; + bool first = true; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + string name = capitalize((*c_iter)->get_name()); + if(first) + first=false; + else + f_types_ << "|"; + f_types_ << name; + } + indent(f_types_) << "deriving (Show,Eq, Typeable, Ord)" << endl; + indent_down(); + + int value = -1; + indent(f_types_) << "instance Enum " << capitalize(tenum->get_name()) << " where" << endl; + indent_up(); + indent(f_types_) << "fromEnum t = case t of" << endl; + indent_up(); + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + string name = capitalize((*c_iter)->get_name()); + + f_types_ << + indent() << name << " -> " << value << endl; + } + indent_down(); + + indent(f_types_) << "toEnum t = case t of" << endl; + indent_up(); + for(c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + string name = capitalize((*c_iter)->get_name()); + + f_types_ << + indent() << value << " -> " << name << endl; + } + indent(f_types_) << "_ -> throw ThriftException" << endl; + indent_down(); + indent_down(); +} + +/** + * Generate a constant value + */ +void t_hs_generator::generate_const(t_const* tconst) { + t_type* type = tconst->get_type(); + string name = decapitalize(tconst->get_name()); + t_const_value* value = tconst->get_value(); + + indent(f_consts_) << name << " :: " << render_hs_type(type, false) << endl; + indent(f_consts_) << name << " = " << render_const_value(type, value) << endl << endl; +} + +/** + * Prints the value of a constant with the given type. Note that type checking + * is NOT performed in this function as it is always run beforehand using the + * validate_types method in main.cc + */ +string t_hs_generator::render_const_value(t_type* type, t_const_value* value) { + type = get_true_type(type); + std::ostringstream out; + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + out << '"' << get_escaped_string(value) << '"'; + break; + case t_base_type::TYPE_BOOL: + out << (value->get_integer() > 0 ? "True" : "False"); + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + out << value->get_integer(); + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() == t_const_value::CV_INTEGER) { + out << value->get_integer(); + } else { + out << value->get_double(); + } + break; + default: + throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + t_enum* tenum = (t_enum*)type; + vector constants = tenum->get_constants(); + vector::iterator c_iter; + int val = -1; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + val = (*c_iter)->get_value(); + } else { + ++val; + } + if(val == value->get_integer()){ + indent(out) << capitalize((*c_iter)->get_name()); + break; + } + } + } else if (type->is_struct() || type->is_xception()) { + string cname = type_name(type); + indent(out) << cname << "{"; + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + const map& val = value->get_map(); + map::const_iterator v_iter; + bool first = true; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + string fname = v_iter->first->get_string(); + if(first) + first=false; + else + out << ","; + out << "f_" << cname << "_" << fname << " = Just (" << render_const_value(field_type, v_iter->second) << ")"; + + } + indent(out) << "}"; + } else if (type->is_map()) { + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + const map& val = value->get_map(); + map::const_iterator v_iter; + out << "(Map.fromList ["; + bool first=true; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + string key = render_const_value(ktype, v_iter->first); + string val = render_const_value(vtype, v_iter->second); + if(first) + first=false; + else + out << ","; + out << "(" << key << ","<< val << ")"; + } + out << "])"; + } else if (type->is_list() || type->is_set()) { + t_type* etype; + + if (type->is_list()) { + etype = ((t_list*) type)->get_elem_type(); + } else { + etype = ((t_set*) type)->get_elem_type(); + } + + const vector& val = value->get_list(); + vector::const_iterator v_iter; + bool first = true; + + if (type->is_set()) + out << "(Set.fromList "; + + out << "["; + + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + if(first) + first=false; + else + out << ","; + out << render_const_value(etype, *v_iter); + } + + out << "]"; + if (type->is_set()) + out << ")"; + } else { + throw "CANNOT GENERATE CONSTANT FOR TYPE: " + type->get_name(); + } + return out.str(); +} + +/** + * Generates a "struct" + */ +void t_hs_generator::generate_struct(t_struct* tstruct) { + generate_hs_struct(tstruct, false); +} + +/** + * Generates a struct definition for a thrift exception. Basically the same + * as a struct, but also has an exception declaration. + * + * @param txception The struct definition + */ +void t_hs_generator::generate_xception(t_struct* txception) { + generate_hs_struct(txception, true); +} + +/** + * Generates a Haskell struct + */ +void t_hs_generator::generate_hs_struct(t_struct* tstruct, + bool is_exception) { + generate_hs_struct_definition(f_types_,tstruct, is_exception,false); +} + +/** + * Generates a struct definition for a thrift data type. + * + * @param tstruct The struct definition + */ +void t_hs_generator::generate_hs_struct_definition(ofstream& out, + t_struct* tstruct, + bool is_exception, + bool helper) { + string tname = type_name(tstruct); + string name = tstruct->get_name(); + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + + indent(out) << "data "< 0) { + out << "{"; + bool first=true; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + if(first) + first=false; + else + out << ","; + string mname = (*m_iter)->get_name(); + out << "f_" << tname << "_" << mname << " :: Maybe " << render_hs_type((*m_iter)->get_type()); + } + out << "}"; + } + + out << " deriving (Show,Eq,Ord,Typeable)" << endl; + if (is_exception) out << "instance Exception " << tname << endl; + generate_hs_struct_writer(out, tstruct); + + generate_hs_struct_reader(out, tstruct); + //f_struct_.close(); +} + + + +/** + * Generates the read method for a struct + */ +void t_hs_generator::generate_hs_struct_reader(ofstream& out, t_struct* tstruct) { + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + string sname = type_name(tstruct); + string str = tmp("_str"); + string t = tmp("_t"); + string id = tmp("_id"); + + indent(out) << "read_" << sname << "_fields iprot rec = do" << endl; + indent_up(); // do + + // Read beginning field marker + indent(out) << "(_," << t <<","<get_key() << " -> "; + out << "if " << t <<" == " << type_to_enum((*f_iter)->get_type()) << " then do" << endl; + indent_up(); // if + indent(out) << "s <- "; + generate_deserialize_field(out, *f_iter,str); + out << endl; + indent(out) << "read_"<get_name()) <<"=Just s}" << endl; + out << + indent() << "else do" << endl; + indent_up(); + indent(out) << "skip iprot "<< t << endl; + indent(out) << "read_"< do" << endl; + indent_up(); + indent(out) << "skip iprot "<get_name()) << "=Nothing"; + } + out << "})" << endl; + indent(out) << "readStructEnd iprot" << endl; + indent(out) << "return rec" << endl; + indent_down(); +} + +void t_hs_generator::generate_hs_struct_writer(ofstream& out, + t_struct* tstruct) { + string name = type_name(tstruct); + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + string str = tmp("_str"); + string f = tmp("_f"); + + indent(out) << + "write_"<get_name(); + indent(out) << + "case f_" << name << "_" << mname << " rec of {Nothing -> return (); Just _v -> do" << endl; + indent_up(); + indent(out) << "writeFieldBegin oprot (\""<< (*f_iter)->get_name()<<"\"," + <get_type())<<"," + <<(*f_iter)->get_key()<<")" << endl; + + // Write field contents + out << indent(); + generate_serialize_field(out, *f_iter, "_v"); + out << endl; + // Write field closer + indent(out) << "writeFieldEnd oprot}" << endl; + indent_down(); + } + + // Write the struct map + out << + indent() << "writeFieldStop oprot" << endl << + indent() << "writeStructEnd oprot" << endl; + + indent_down(); +} + +/** + * Generates a thrift service. + * + * @param tservice The service definition + */ +void t_hs_generator::generate_service(t_service* tservice) { + string f_service_name = get_out_dir()+capitalize(service_name_)+".hs"; + f_service_.open(f_service_name.c_str()); + + f_service_ << + hs_autogen_comment() << endl << + "module " << capitalize(service_name_) << " where" << endl << + hs_imports() << endl; + + + if(tservice->get_extends()){ + f_service_ << + "import qualified " << capitalize(tservice->get_extends()->get_name()) << endl; + } + + + f_service_ << + "import " << capitalize(program_name_) << "_Types" << endl << + "import qualified " << capitalize(service_name_) << "_Iface as Iface" << endl; + + + // Generate the three main parts of the service + generate_service_helpers(tservice); + generate_service_interface(tservice); + generate_service_client(tservice); + generate_service_server(tservice); + + + // Close service file + f_service_.close(); +} + +/** + * Generates helper functions for a service. + * + * @param tservice The service to generate a header definition for + */ +void t_hs_generator::generate_service_helpers(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + indent(f_service_) << + "-- HELPER FUNCTIONS AND STRUCTURES --" << endl << endl; + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* ts = (*f_iter)->get_arglist(); + generate_hs_struct_definition(f_service_,ts, false); + generate_hs_function_helpers(*f_iter); + } +} + +/** + * Generates a struct and helpers for a function. + * + * @param tfunction The function + */ +void t_hs_generator::generate_hs_function_helpers(t_function* tfunction) { + t_struct result(program_, decapitalize(tfunction->get_name()) + "_result"); + t_field success(tfunction->get_returntype(), "success", 0); + if (!tfunction->get_returntype()->is_void()) { + result.append(&success); + } + + t_struct* xs = tfunction->get_xceptions(); + const vector& fields = xs->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + result.append(*f_iter); + } + generate_hs_struct_definition(f_service_,&result, false); +} + +/** + * Generates a service interface definition. + * + * @param tservice The service to generate a header definition for + */ +void t_hs_generator::generate_service_interface(t_service* tservice) { + string f_iface_name = get_out_dir()+capitalize(service_name_)+"_Iface.hs"; + f_iface_.open(f_iface_name.c_str()); + indent(f_iface_) << "module " << capitalize(service_name_) << "_Iface where" << endl; + + indent(f_iface_) << + hs_imports() << endl << + "import " << capitalize(program_name_) << "_Types" << endl << + endl; + + if (tservice->get_extends() != NULL) { + string extends = type_name(tservice->get_extends()); + indent(f_iface_) << "import " << extends <<"_Iface" << endl; + indent(f_iface_) << "class "<< extends << "_Iface a => " << capitalize(service_name_) << "_Iface a where" << endl; + } else { + f_iface_ << indent() << "class " << capitalize(service_name_) << "_Iface a where" << endl; + } + indent_up(); + + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string ft = function_type(*f_iter,true,true,true); + f_iface_ << + indent() << decapitalize((*f_iter)->get_name()) << " :: a -> " << ft << endl; + } + indent_down(); + f_iface_.close(); + +} + +/** + * Generates a service client definition. Note that in Haskell, the client doesn't implement iface. This is because + * The client does not (and should not have to) deal with arguments being Nothing. + * + * @param tservice The service to generate a server for. + */ +void t_hs_generator::generate_service_client(t_service* tservice) { + string f_client_name = get_out_dir()+capitalize(service_name_)+"_Client.hs"; + f_client_.open(f_client_name.c_str()); + + vector functions = tservice->get_functions(); + vector::const_iterator f_iter; + + string extends = ""; + string exports=""; + bool first = true; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + if(first) + first=false; + else + exports+=","; + string funname = (*f_iter)->get_name(); + exports+=funname; + } + indent(f_client_) << "module " << capitalize(service_name_) << "_Client("<get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + indent(f_client_) << "import " << extends << "_Client" << endl; + } + indent(f_client_) << "import Data.IORef" << endl; + indent(f_client_) << hs_imports() << endl; + indent(f_client_) << "import " << capitalize(program_name_) << "_Types" << endl; + indent(f_client_) << "import " << capitalize(service_name_) << endl; + // DATS RITE A GLOBAL VAR + indent(f_client_) << "seqid = newIORef 0" << endl; + + + // Generate client method implementations + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* arg_struct = (*f_iter)->get_arglist(); + const vector& fields = arg_struct->get_members(); + vector::const_iterator fld_iter; + string funname = (*f_iter)->get_name(); + + string fargs = ""; + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + fargs+= " arg_" + decapitalize((*fld_iter)->get_name()); + } + + // Open function + indent(f_client_) << funname << " (ip,op)" << fargs << " = do" << endl; + indent_up(); + indent(f_client_) << "send_" << funname << " op" << fargs; + + f_client_ << endl; + + if (!(*f_iter)->is_oneway()) { + f_client_ << indent(); + f_client_ << + "recv_" << funname << " ip" << endl; + } + indent_down(); + + indent(f_client_) << + "send_" << funname << " op" << fargs << " = do" << endl; + indent_up(); + indent(f_client_) << "seq <- seqid" << endl; + indent(f_client_) << "seqn <- readIORef seq" << endl; + std::string argsname = capitalize((*f_iter)->get_name() + "_args"); + + // Serialize the request header + f_client_ << + indent() << "writeMessageBegin op (\"" << (*f_iter)->get_name() << "\", M_CALL, seqn)" << endl; + f_client_ << indent() << "write_" << argsname << " op ("<get_name() << "=Just arg_" << (*fld_iter)->get_name(); + } + f_client_ << "})" << endl; + + // Write to the stream + f_client_ << + indent() << "writeMessageEnd op" << endl << + indent() << "tFlush (getTransport op)" << endl; + + indent_down(); + + if (!(*f_iter)->is_oneway()) { + std::string resultname = capitalize((*f_iter)->get_name() + "_result"); + t_struct noargs(program_); + + std::string funname = string("recv_") + (*f_iter)->get_name(); + + t_function recv_function((*f_iter)->get_returntype(), + funname, + &noargs); + // Open function + f_client_ << + indent() << funname << " ip = do" << endl; + indent_up(); // fun + + // TODO(mcslee): Validate message reply here, seq ids etc. + + f_client_ << + indent() << "(fname, mtype, rseqid) <- readMessageBegin ip" << endl; + f_client_ << + indent() << "if mtype == M_EXCEPTION then do" << endl << + indent() << " x <- readAppExn ip" << endl << + indent() << " readMessageEnd ip" << endl; + f_client_ << + indent() << " throw x" << endl; + f_client_ << + indent() << " else return ()" << endl; + + t_struct* xs = (*f_iter)->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + + f_client_ << + indent() << "res <- read_" << resultname << " ip" << endl; + f_client_ << + indent() << "readMessageEnd ip" << endl; + + // Careful, only return _result if not a void function + if (!(*f_iter)->get_returntype()->is_void()) { + f_client_ << + indent() << "case f_" << resultname << "_success res of" << endl; + indent_up(); // case + indent(f_client_) << "Just v -> return v" << endl; + indent(f_client_) << "Nothing -> do" << endl; + indent_up(); // none + } + + + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_client_ << + indent() << "case f_"<< resultname << "_" << (*x_iter)->get_name() << " res of" << endl; + indent_up(); //case + indent(f_client_) << "Nothing -> return ()" << endl; + indent(f_client_) << "Just _v -> throw _v" << endl; + indent_down(); //-case + } + + // Careful, only return _result if not a void function + if ((*f_iter)->get_returntype()->is_void()) { + indent(f_client_) << + "return ()" << endl; + } else { + f_client_ << + indent() << "throw (AppExn AE_MISSING_RESULT \"" << (*f_iter)->get_name() << " failed: unknown result\")" << endl; + indent_down(); //-none + indent_down(); //-case + } + + // Close function + indent_down(); //-fun + } + } + f_client_.close(); + + +} + +/** + * Generates a service server definition. + * + * @param tservice The service to generate a server for. + */ +void t_hs_generator::generate_service_server(t_service* tservice) { + // Generate the dispatch methods + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + // Generate the process subfunctions + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + generate_process_function(tservice, *f_iter); + } + + + indent(f_service_) << "proc handler (iprot,oprot) (name,typ,seqid) = case name of" << endl; + indent_up(); + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string fname = (*f_iter)->get_name(); + indent(f_service_) << "\""< process_" << decapitalize(fname) << " (seqid,iprot,oprot,handler)" << endl; + } + indent(f_service_) << "_ -> "; + if(tservice->get_extends() != NULL){ + f_service_ << type_name(tservice->get_extends()) << ".proc handler (iprot,oprot) (name,typ,seqid)" << endl; + } else { + f_service_ << "do" << endl; + indent_up(); + indent(f_service_) << "skip iprot T_STRUCT" << endl; + indent(f_service_) << "readMessageEnd iprot" << endl; + indent(f_service_) << "writeMessageBegin oprot (name,M_EXCEPTION,seqid)" << endl; + indent(f_service_) << "writeAppExn oprot (AppExn AE_UNKNOWN_METHOD (\"Unknown function \" ++ name))" << endl; + indent(f_service_) << "writeMessageEnd oprot" << endl; + indent(f_service_) << "tFlush (getTransport oprot)" << endl; + indent_down(); + } + indent_down(); + + // Generate the server implementation + indent(f_service_) << + "process handler (iprot, oprot) = do" << endl; + indent_up(); + + f_service_ << + indent() << "(name, typ, seqid) <- readMessageBegin iprot" << endl; + f_service_ << indent() << "proc handler (iprot,oprot) (name,typ,seqid)" << endl; + indent(f_service_) << "return True" << endl; + indent_down(); + +} + +/** + * Generates a process function definition. + * + * @param tfunction The function to write a dispatcher for + */ +void t_hs_generator::generate_process_function(t_service* tservice, + t_function* tfunction) { + // Open function + indent(f_service_) << + "process_" << tfunction->get_name() << " (seqid, iprot, oprot, handler) = do" << endl; + indent_up(); + + string argsname = capitalize(tfunction->get_name()) + "_args"; + string resultname = capitalize(tfunction->get_name()) + "_result"; + + // Generate the function call + t_struct* arg_struct = tfunction->get_arglist(); + const std::vector& fields = arg_struct->get_members(); + vector::const_iterator f_iter; + + + f_service_ << + indent() << "args <- read_" << argsname << " iprot" << endl; + f_service_ << + indent() << "readMessageEnd iprot" << endl; + + t_struct* xs = tfunction->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + int n = xceptions.size(); + if (!tfunction->is_oneway()){ + if(!tfunction->get_returntype()->is_void()){ + n++; + } + indent(f_service_) << "rs <- return (" << resultname; + + for(int i=0; i 0) { + for(unsigned int i=0;iis_oneway() && !tfunction->get_returntype()->is_void()){ + f_service_ << "res <- "; + } + f_service_ << "Iface." << tfunction->get_name() << " handler"; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + f_service_ << " (f_" << argsname << "_" << (*f_iter)->get_name() << " args)"; + } + + + if (!tfunction->is_oneway() && !tfunction->get_returntype()->is_void()){ + f_service_ << endl; + indent(f_service_) << "return rs{f_"<is_oneway()){ + f_service_ << endl; + indent(f_service_) << "return rs"; + } + f_service_ << ")" << endl; + indent_down(); + + if (xceptions.size() > 0 && !tfunction->is_oneway()) { + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + indent(f_service_) << "(\\e -> " <is_oneway()){ + f_service_ << + indent() << "return rs{f_"<get_name() << " =Just e}"; + } else { + indent(f_service_) << "return ()"; + } + f_service_ << "))" << endl; + indent_down(); + indent_down(); + } + } + + + + // Shortcut out here for oneway functions + if (tfunction->is_oneway()) { + f_service_ << + indent() << "return ()" << endl; + indent_down(); + return; + } + + f_service_ << + indent() << "writeMessageBegin oprot (\"" << tfunction->get_name() << "\", M_REPLY, seqid);" << endl << + indent() << "write_"<get_type(); + generate_deserialize_type(out,type); +} + + +/** + * Deserializes a field of any type. + */ +void t_hs_generator::generate_deserialize_type(ofstream &out, + t_type* type){ + type = get_true_type(type); + + if (type->is_void()) { + throw "CANNOT GENERATE DESERIALIZE CODE FOR void TYPE"; + } + + + if (type->is_struct() || type->is_xception()) { + generate_deserialize_struct(out, + (t_struct*)type); + } else if (type->is_container()) { + generate_deserialize_container(out, type); + } else if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "compiler error: cannot serialize void field in a struct"; + break; + case t_base_type::TYPE_STRING: + out << "readString"; + break; + case t_base_type::TYPE_BOOL: + out << "readBool"; + break; + case t_base_type::TYPE_BYTE: + out << "readByte"; + break; + case t_base_type::TYPE_I16: + out << "readI16"; + break; + case t_base_type::TYPE_I32: + out << "readI32"; + break; + case t_base_type::TYPE_I64: + out << "readI64"; + break; + case t_base_type::TYPE_DOUBLE: + out << "readDouble"; + break; + default: + throw "compiler error: no PHP name for base type " + t_base_type::t_base_name(tbase); + } + out << " iprot"; + } else if (type->is_enum()) { + string ename = capitalize(type->get_name()); + out << "(do {i <- readI32 iprot; return (toEnum i :: " << ename << ")})"; + } else { + printf("DO NOT KNOW HOW TO DESERIALIZE TYPE '%s'\n", + type->get_name().c_str()); + } +} + + +/** + * Generates an unserializer for a struct, calling read() + */ +void t_hs_generator::generate_deserialize_struct(ofstream &out, + t_struct* tstruct) { + string name = capitalize(tstruct->get_name()); + out << "(read_" << name << " iprot)"; + +} + +/** + * Serialize a container by writing out the header followed by + * data and then a footer. + */ +void t_hs_generator::generate_deserialize_container(ofstream &out, + t_type* ttype) { + string size = tmp("_size"); + string ktype = tmp("_ktype"); + string vtype = tmp("_vtype"); + string etype = tmp("_etype"); + string con = tmp("_con"); + + t_field fsize(g_type_i32, size); + t_field fktype(g_type_byte, ktype); + t_field fvtype(g_type_byte, vtype); + t_field fetype(g_type_byte, etype); + + // Declare variables, read header + if (ttype->is_map()) { + out << "(let {f 0 = return []; f n = do {k <- "; + generate_deserialize_type(out,((t_map*)ttype)->get_key_type()); + out << "; v <- "; + generate_deserialize_type(out,((t_map*)ttype)->get_val_type()); + out << ";r <- f (n-1); return $ (k,v):r}} in do {("<is_set()) { + out << "(let {f 0 = return []; f n = do {v <- "; + generate_deserialize_type(out,((t_map*)ttype)->get_key_type()); + out << ";r <- f (n-1); return $ v:r}} in do {("<is_list()) { + out << "(let {f 0 = return []; f n = do {v <- "; + generate_deserialize_type(out,((t_map*)ttype)->get_key_type()); + out << ";r <- f (n-1); return $ v:r}} in do {("<get_type()); + + // Do nothing for void types + if (type->is_void()) { + throw "CANNOT GENERATE SERIALIZE CODE FOR void TYPE: " + + tfield->get_name(); + } + + if(name.length() == 0){ + name = decapitalize(tfield->get_name()); + } + + if (type->is_struct() || type->is_xception()) { + generate_serialize_struct(out, + (t_struct*)type, + name); + } else if (type->is_container()) { + generate_serialize_container(out, + type, + name); + } else if (type->is_base_type() || type->is_enum()) { + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw + "compiler error: cannot serialize void field in a struct: " + name; + break; + case t_base_type::TYPE_STRING: + out << "writeString oprot " << name; + break; + case t_base_type::TYPE_BOOL: + out << "writeBool oprot " << name; + break; + case t_base_type::TYPE_BYTE: + out << "writeByte oprot " << name; + break; + case t_base_type::TYPE_I16: + out << "writeI16 oprot " << name; + break; + case t_base_type::TYPE_I32: + out << "writeI32 oprot " << name; + break; + case t_base_type::TYPE_I64: + out << "writeI64 oprot " << name; + break; + case t_base_type::TYPE_DOUBLE: + out << "writeDouble oprot " << name; + break; + default: + throw "compiler error: no hs name for base type " + t_base_type::t_base_name(tbase); + } + + } else if (type->is_enum()) { + string ename = capitalize(type->get_name()); + out << "writeI32 oprot (fromEnum "<< name << ")"; + } + + } else { + printf("DO NOT KNOW HOW TO SERIALIZE FIELD '%s' TYPE '%s'\n", + tfield->get_name().c_str(), + type->get_name().c_str()); + } +} + +/** + * Serializes all the members of a struct. + * + * @param tstruct The struct to serialize + * @param prefix String prefix to attach to all fields + */ +void t_hs_generator::generate_serialize_struct(ofstream &out, + t_struct* tstruct, + string prefix) { + out << "write_" << type_name(tstruct) << " oprot " << prefix; +} + +void t_hs_generator::generate_serialize_container(ofstream &out, + t_type* ttype, + string prefix) { + if (ttype->is_map()) { + string k = tmp("_kiter"); + string v = tmp("_viter"); + out << "(let {f [] = return (); f (("<get_key_type())<<","<< type_to_enum(((t_map*)ttype)->get_val_type())<<",Map.size " << prefix << "); f (Map.toList " << prefix << ");writeMapEnd oprot})"; + } else if (ttype->is_set()) { + string v = tmp("_viter"); + out << "(let {f [] = return (); f ("<get_elem_type())<<",Set.size " << prefix << "); f (Set.toList " << prefix << ");writeSetEnd oprot})"; + } else if (ttype->is_list()) { + string v = tmp("_viter"); + out << "(let {f [] = return (); f ("<get_elem_type())<<",length " << prefix << "); f " << prefix << ";writeListEnd oprot})"; + } + +} + +/** + * Serializes the members of a map. + * + */ +void t_hs_generator::generate_serialize_map_element(ofstream &out, + t_map* tmap, + string kiter, + string viter) { + t_field kfield(tmap->get_key_type(), kiter); + out << "do {"; + generate_serialize_field(out, &kfield); + out << ";"; + t_field vfield(tmap->get_val_type(), viter); + generate_serialize_field(out, &vfield); + out << "}"; +} + +/** + * Serializes the members of a set. + */ +void t_hs_generator::generate_serialize_set_element(ofstream &out, + t_set* tset, + string iter) { + t_field efield(tset->get_elem_type(), iter); + generate_serialize_field(out, &efield); +} + +/** + * Serializes the members of a list. + */ +void t_hs_generator::generate_serialize_list_element(ofstream &out, + t_list* tlist, + string iter) { + t_field efield(tlist->get_elem_type(), iter); + generate_serialize_field(out, &efield); +} + + +string t_hs_generator::function_type(t_function* tfunc, bool options, bool io, bool method){ + string result=""; + + const vector& fields = tfunc->get_arglist()->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if(options) result += "Maybe "; + result += render_hs_type((*f_iter)->get_type(), options); + result += " -> "; + } + if(fields.empty() && !method){ + result += "() -> "; + } + if(io) result += "IO "; + result += render_hs_type(tfunc->get_returntype(), io); + return result; +} + + +string t_hs_generator::type_name(t_type* ttype) { + string prefix = ""; + t_program* program = ttype->get_program(); + if (program != NULL && program != program_) { + if (!ttype->is_service()) { + prefix = capitalize(program->get_name()) + "_Types."; + } + } + + string name = ttype->get_name(); + if(ttype->is_service()){ + name = capitalize(name); + } else { + name = capitalize(name); + } + return prefix + name; +} + +/** + * Converts the parse type to a Protocol.t_type enum + */ +string t_hs_generator::type_to_enum(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + return "T_VOID"; + case t_base_type::TYPE_STRING: + return "T_STRING"; + case t_base_type::TYPE_BOOL: + return "T_BOOL"; + case t_base_type::TYPE_BYTE: + return "T_BYTE"; + case t_base_type::TYPE_I16: + return "T_I16"; + case t_base_type::TYPE_I32: + return "T_I32"; + case t_base_type::TYPE_I64: + return "T_I64"; + case t_base_type::TYPE_DOUBLE: + return "T_DOUBLE"; + } + } else if (type->is_enum()) { + return "T_I32"; + } else if (type->is_struct() || type->is_xception()) { + return "T_STRUCT"; + } else if (type->is_map()) { + return "T_MAP"; + } else if (type->is_set()) { + return "T_SET"; + } else if (type->is_list()) { + return "T_LIST"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + +/** + * Converts the parse type to an haskell type + */ +string t_hs_generator::render_hs_type(t_type* type, bool needs_parens) { + type = get_true_type(type); + string type_repr; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + return "()"; + case t_base_type::TYPE_STRING: + return "String"; + case t_base_type::TYPE_BOOL: + return "Bool"; + case t_base_type::TYPE_BYTE: + return "Int"; + case t_base_type::TYPE_I16: + return "Int"; + case t_base_type::TYPE_I32: + return "Int"; + case t_base_type::TYPE_I64: + return "Int64"; + case t_base_type::TYPE_DOUBLE: + return "Double"; + } + } else if (type->is_enum()) { + return capitalize(((t_enum*)type)->get_name()); + } else if (type->is_struct() || type->is_xception()) { + return type_name((t_struct*)type); + } else if (type->is_map()) { + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + + type_repr = "Map.Map " + render_hs_type(ktype, true) + " " + render_hs_type(vtype, true); + } else if (type->is_set()) { + t_type* etype = ((t_set*)type)->get_elem_type(); + + type_repr = "Set.Set " + render_hs_type(etype, true) ; + } else if (type->is_list()) { + t_type* etype = ((t_list*)type)->get_elem_type(); + return "[" + render_hs_type(etype, false) + "]"; + } else { + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); + } + + return needs_parens ? "(" + type_repr + ")" : type_repr; +} + + +THRIFT_REGISTER_GENERATOR(hs, "Haskell", ""); diff --git a/compiler/cpp/src/generate/t_html_generator.cc b/compiler/cpp/src/generate/t_html_generator.cc new file mode 100644 index 00000000..ad1c4cbc --- /dev/null +++ b/compiler/cpp/src/generate/t_html_generator.cc @@ -0,0 +1,637 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include "t_generator.h" +#include "platform.h" +using namespace std; + + +/** + * HTML code generator + * + * mostly copy/pasting/tweaking from mcslee's work. + */ +class t_html_generator : public t_generator { + public: + t_html_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_generator(program) + { + out_dir_base_ = "gen-html"; + escape_.clear(); + escape_['&'] = "&"; + escape_['<'] = "<"; + escape_['>'] = ">"; + escape_['"'] = """; + escape_['\''] = "'"; + } + + void generate_program(); + void generate_program_toc(); + void generate_program_toc_row(t_program* tprog); + void generate_program_toc_rows(t_program* tprog, + std::vector& finished); + void generate_index(); + void generate_css(); + + /** + * Program-level generation functions + */ + + void generate_typedef (t_typedef* ttypedef); + void generate_enum (t_enum* tenum); + void generate_const (t_const* tconst); + void generate_struct (t_struct* tstruct); + void generate_service (t_service* tservice); + void generate_xception(t_struct* txception); + + void print_doc (t_doc* tdoc); + int print_type (t_type* ttype); + void print_const_value(t_const_value* tvalue); + + std::ofstream f_out_; +}; + +/** + * Emits the Table of Contents links at the top of the module's page + */ +void t_html_generator::generate_program_toc() { + f_out_ << "" + << "" << endl; + generate_program_toc_row(program_); + f_out_ << "
ModuleServicesData typesConstants
" << endl; +} + + +/** + * Recurses through from the provided program and generates a ToC row + * for each discovered program exactly once by maintaining the list of + * completed rows in 'finished' + */ +void t_html_generator::generate_program_toc_rows(t_program* tprog, + std::vector& finished) { + for (vector::iterator iter = finished.begin(); + iter != finished.end(); iter++) { + if (tprog->get_path() == (*iter)->get_path()) { + return; + } + } + finished.push_back(tprog); + generate_program_toc_row(tprog); + vector includes = tprog->get_includes(); + for (vector::iterator iter = includes.begin(); + iter != includes.end(); iter++) { + generate_program_toc_rows(*iter, finished); + } +} + +/** + * Emits the Table of Contents links at the top of the module's page + */ +void t_html_generator::generate_program_toc_row(t_program* tprog) { + string fname = tprog->get_name() + ".html"; + f_out_ << "" << endl << "" << tprog->get_name() << ""; + if (!tprog->get_services().empty()) { + vector services = tprog->get_services(); + vector::iterator sv_iter; + for (sv_iter = services.begin(); sv_iter != services.end(); ++sv_iter) { + string name = get_service_name(*sv_iter); + f_out_ << "" << name + << "
" << endl; + f_out_ << "
    " << endl; + map fn_html; + vector functions = (*sv_iter)->get_functions(); + vector::iterator fn_iter; + for (fn_iter = functions.begin(); fn_iter != functions.end(); ++fn_iter) { + string fn_name = (*fn_iter)->get_name(); + string html = "
  • " + fn_name + "
    • "; + fn_html.insert(pair(fn_name, html)); + } + for (map::iterator html_iter = fn_html.begin(); + html_iter != fn_html.end(); html_iter++) { + f_out_ << html_iter->second << endl; + } + f_out_ << "
" << endl; + } + } + f_out_ << "" << endl << ""; + map data_types; + if (!tprog->get_enums().empty()) { + vector enums = tprog->get_enums(); + vector::iterator en_iter; + for (en_iter = enums.begin(); en_iter != enums.end(); ++en_iter) { + string name = (*en_iter)->get_name(); + // f_out_ << "" << name + // << "
" << endl; + string html = "" + name + + ""; + data_types.insert(pair(name, html)); + } + } + if (!tprog->get_typedefs().empty()) { + vector typedefs = tprog->get_typedefs(); + vector::iterator td_iter; + for (td_iter = typedefs.begin(); td_iter != typedefs.end(); ++td_iter) { + string name = (*td_iter)->get_symbolic(); + // f_out_ << "" << name + // << "
" << endl; + string html = "" + name + + ""; + data_types.insert(pair(name, html)); + } + } + if (!tprog->get_objects().empty()) { + vector objects = tprog->get_objects(); + vector::iterator o_iter; + for (o_iter = objects.begin(); o_iter != objects.end(); ++o_iter) { + string name = (*o_iter)->get_name(); + //f_out_ << "" << name + //<< "
" << endl; + string html = "" + name + + ""; + data_types.insert(pair(name, html)); + } + } + for (map::iterator dt_iter = data_types.begin(); + dt_iter != data_types.end(); dt_iter++) { + f_out_ << dt_iter->second << "
" << endl; + } + f_out_ << "" << endl << ""; + if (!tprog->get_consts().empty()) { + map const_html; + vector consts = tprog->get_consts(); + vector::iterator con_iter; + for (con_iter = consts.begin(); con_iter != consts.end(); ++con_iter) { + string name = (*con_iter)->get_name(); + string html ="" + name + ""; + const_html.insert(pair(name, html)); + } + for (map::iterator con_iter = const_html.begin(); + con_iter != const_html.end(); con_iter++) { + f_out_ << con_iter->second << "
" << endl; + } + } + f_out_ << "" << endl << ""; +} + +/** + * Prepares for file generation by opening up the necessary file output + * stream. + */ +void t_html_generator::generate_program() { + // Make output directory + MKDIR(get_out_dir().c_str()); + string fname = get_out_dir() + program_->get_name() + ".html"; + f_out_.open(fname.c_str()); + f_out_ << "" << endl; + f_out_ << "" + << endl; + f_out_ << "Thrift module: " << program_->get_name() + << "" << endl << "

Thrift module: " + << program_->get_name() << "

" << endl; + + print_doc(program_); + + generate_program_toc(); + + if (!program_->get_consts().empty()) { + f_out_ << "

Constants

" << endl; + vector consts = program_->get_consts(); + f_out_ << ""; + f_out_ << "" << endl; + generate_consts(consts); + f_out_ << "
ConstantTypeValue
"; + } + + if (!program_->get_enums().empty()) { + f_out_ << "

Enumerations

" << endl; + // Generate enums + vector enums = program_->get_enums(); + vector::iterator en_iter; + for (en_iter = enums.begin(); en_iter != enums.end(); ++en_iter) { + generate_enum(*en_iter); + } + } + + if (!program_->get_typedefs().empty()) { + f_out_ << "

Type declarations

" << endl; + // Generate typedefs + vector typedefs = program_->get_typedefs(); + vector::iterator td_iter; + for (td_iter = typedefs.begin(); td_iter != typedefs.end(); ++td_iter) { + generate_typedef(*td_iter); + } + } + + if (!program_->get_objects().empty()) { + f_out_ << "

Data structures

" << endl; + // Generate structs and exceptions in declared order + vector objects = program_->get_objects(); + vector::iterator o_iter; + for (o_iter = objects.begin(); o_iter != objects.end(); ++o_iter) { + if ((*o_iter)->is_xception()) { + generate_xception(*o_iter); + } else { + generate_struct(*o_iter); + } + } + } + + if (!program_->get_services().empty()) { + f_out_ << "

Services

" << endl; + // Generate services + vector services = program_->get_services(); + vector::iterator sv_iter; + for (sv_iter = services.begin(); sv_iter != services.end(); ++sv_iter) { + service_name_ = get_service_name(*sv_iter); + generate_service(*sv_iter); + } + } + + f_out_ << "" << endl; + f_out_.close(); + + generate_index(); + generate_css(); +} + +/** + * Emits the index.html file for the recursive set of Thrift programs + */ +void t_html_generator::generate_index() { + string index_fname = get_out_dir() + "index.html"; + f_out_.open(index_fname.c_str()); + f_out_ << "" << endl; + f_out_ << "" + << endl; + f_out_ << "All Thrift declarations" + << endl << "

All Thrift declarations

" << endl; + f_out_ << "" + << "" << endl; + vector programs; + generate_program_toc_rows(program_, programs); + f_out_ << "
ModuleServicesData typesConstants
" << endl; + f_out_ << "" << endl; + f_out_.close(); +} + +void t_html_generator::generate_css() { + string css_fname = get_out_dir() + "style.css"; + f_out_.open(css_fname.c_str()); + f_out_ << "/* Auto-generated CSS for generated Thrift docs */" << endl; + f_out_ << + "body { font-family: Tahoma, sans-serif; }" << endl; + f_out_ << + "pre { background-color: #dddddd; padding: 6px; }" << endl; + f_out_ << + "h3,h4 { padding-top: 0px; margin-top: 0px; }" << endl; + f_out_ << + "div.definition { border: 1px solid gray; margin: 10px; padding: 10px; }" << endl; + f_out_ << + "div.extends { margin: -0.5em 0 1em 5em }" << endl; + f_out_ << + "table { border: 1px solid grey; border-collapse: collapse; }" << endl; + f_out_ << + "td { border: 1px solid grey; padding: 1px 6px; vertical-align: top; }" << endl; + f_out_ << + "th { border: 1px solid black; background-color: #bbbbbb;" << endl << + " text-align: left; padding: 1px 6px; }" << endl; + f_out_.close(); +} + +/** + * If the provided documentable object has documentation attached, this + * will emit it to the output stream in HTML format. + */ +void t_html_generator::print_doc(t_doc* tdoc) { + if (tdoc->has_doc()) { + string doc = tdoc->get_doc(); + size_t index; + while ((index = doc.find_first_of("\r\n")) != string::npos) { + if (index == 0) { + f_out_ << "

" << endl; + } else { + f_out_ << doc.substr(0, index) << endl; + } + if (index + 1 < doc.size() && doc.at(index) != doc.at(index + 1) && + (doc.at(index + 1) == '\r' || doc.at(index + 1) == '\n')) { + index++; + } + doc = doc.substr(index + 1); + } + f_out_ << doc << "
"; + } +} + +/** + * Prints out the provided type in HTML + */ +int t_html_generator::print_type(t_type* ttype) { + int len = 0; + f_out_ << ""; + if (ttype->is_container()) { + if (ttype->is_list()) { + f_out_ << "list<"; + len = 6 + print_type(((t_list*)ttype)->get_elem_type()); + f_out_ << ">"; + } else if (ttype->is_set()) { + f_out_ << "set<"; + len = 5 + print_type(((t_set*)ttype)->get_elem_type()); + f_out_ << ">"; + } else if (ttype->is_map()) { + f_out_ << "map<"; + len = 5 + print_type(((t_map*)ttype)->get_key_type()); + f_out_ << ", "; + len += print_type(((t_map*)ttype)->get_val_type()); + f_out_ << ">"; + } + } else if (ttype->is_base_type()) { + f_out_ << ttype->get_name(); + len = ttype->get_name().size(); + } else { + string prog_name = ttype->get_program()->get_name(); + string type_name = ttype->get_name(); + f_out_ << "is_typedef()) { + f_out_ << "Typedef_"; + } else if (ttype->is_struct() || ttype->is_xception()) { + f_out_ << "Struct_"; + } else if (ttype->is_enum()) { + f_out_ << "Enum_"; + } else if (ttype->is_service()) { + f_out_ << "Svc_"; + } + f_out_ << type_name << "\">"; + len = type_name.size(); + if (ttype->get_program() != program_) { + f_out_ << prog_name << "."; + len += prog_name.size() + 1; + } + f_out_ << type_name << ""; + } + f_out_ << ""; + return len; +} + +/** + * Prints out an HTML representation of the provided constant value + */ +void t_html_generator::print_const_value(t_const_value* tvalue) { + bool first = true; + switch (tvalue->get_type()) { + case t_const_value::CV_INTEGER: + f_out_ << tvalue->get_integer(); + break; + case t_const_value::CV_DOUBLE: + f_out_ << tvalue->get_double(); + break; + case t_const_value::CV_STRING: + f_out_ << '"' << get_escaped_string(tvalue) << '"'; + break; + case t_const_value::CV_MAP: + { + f_out_ << "{ "; + map map_elems = tvalue->get_map(); + map::iterator map_iter; + for (map_iter = map_elems.begin(); map_iter != map_elems.end(); + map_iter++) { + if (!first) { + f_out_ << ", "; + } + first = false; + print_const_value(map_iter->first); + f_out_ << " = "; + print_const_value(map_iter->second); + } + f_out_ << " }"; + } + break; + case t_const_value::CV_LIST: + { + f_out_ << "{ "; + vector list_elems = tvalue->get_list();; + vector::iterator list_iter; + for (list_iter = list_elems.begin(); list_iter != list_elems.end(); + list_iter++) { + if (!first) { + f_out_ << ", "; + } + first = false; + print_const_value(*list_iter); + } + f_out_ << " }"; + } + break; + default: + f_out_ << "UNKNOWN"; + break; + } +} + +/** + * Generates a typedef. + * + * @param ttypedef The type definition + */ +void t_html_generator::generate_typedef(t_typedef* ttypedef) { + string name = ttypedef->get_name(); + f_out_ << "

"; + f_out_ << "

Typedef: " << name + << "

" << endl; + f_out_ << "

Base type: "; + print_type(ttypedef->get_type()); + f_out_ << "

" << endl; + print_doc(ttypedef); + f_out_ << "
" << endl; +} + +/** + * Generates code for an enumerated type. + * + * @param tenum The enumeration + */ +void t_html_generator::generate_enum(t_enum* tenum) { + string name = tenum->get_name(); + f_out_ << "
"; + f_out_ << "

Enumeration: " << name + << "

" << endl; + print_doc(tenum); + vector values = tenum->get_constants(); + vector::iterator val_iter; + f_out_ << "
" << endl; + for (val_iter = values.begin(); val_iter != values.end(); ++val_iter) { + f_out_ << "" << endl; + } + f_out_ << "
"; + f_out_ << (*val_iter)->get_name(); + f_out_ << ""; + f_out_ << (*val_iter)->get_value(); + f_out_ << "
" << endl; +} + +/** + * Generates a constant value + */ +void t_html_generator::generate_const(t_const* tconst) { + string name = tconst->get_name(); + f_out_ << "" << name + << ""; + print_type(tconst->get_type()); + f_out_ << ""; + print_const_value(tconst->get_value()); + f_out_ << ""; + if (tconst->has_doc()) { + f_out_ << "
"; + print_doc(tconst); + f_out_ << "
"; + } +} + +/** + * Generates a struct definition for a thrift data type. + * + * @param tstruct The struct definition + */ +void t_html_generator::generate_struct(t_struct* tstruct) { + string name = tstruct->get_name(); + f_out_ << "
"; + f_out_ << "

"; + if (tstruct->is_xception()) { + f_out_ << "Exception: "; + } else { + f_out_ << "Struct: "; + } + f_out_ << name << "

" << endl; + vector members = tstruct->get_members(); + vector::iterator mem_iter = members.begin(); + f_out_ << ""; + f_out_ << "" + << endl; + for ( ; mem_iter != members.end(); mem_iter++) { + f_out_ << "" << endl; + } + f_out_ << "
FieldTypeRequiredDefault value
" << (*mem_iter)->get_name() << ""; + print_type((*mem_iter)->get_type()); + f_out_ << ""; + if ((*mem_iter)->get_req() != t_field::T_OPTIONAL) { + f_out_ << "yes"; + } else { + f_out_ << "no"; + } + f_out_ << ""; + t_const_value* default_val = (*mem_iter)->get_value(); + if (default_val != NULL) { + print_const_value(default_val); + } + f_out_ << "

"; + print_doc(tstruct); + f_out_ << "
"; +} + +/** + * Exceptions are special structs + * + * @param tstruct The struct definition + */ +void t_html_generator::generate_xception(t_struct* txception) { + generate_struct(txception); +} + +/** + * Generates the HTML block for a Thrift service. + * + * @param tservice The service definition + */ +void t_html_generator::generate_service(t_service* tservice) { + f_out_ << "

Service: " + << service_name_ << "

" << endl; + + if (tservice->get_extends()) { + f_out_ << "
extends "; + print_type(tservice->get_extends()); + f_out_ << "
\n"; + } + print_doc(tservice); + vector functions = tservice->get_functions(); + vector::iterator fn_iter = functions.begin(); + for ( ; fn_iter != functions.end(); fn_iter++) { + string fn_name = (*fn_iter)->get_name(); + f_out_ << "
"; + f_out_ << "

Function: " << service_name_ << "." << fn_name + << "

" << endl; + f_out_ << "
";
+    int offset = print_type((*fn_iter)->get_returntype());
+    bool first = true;
+    f_out_ << " " << fn_name << "(";
+    offset += fn_name.size() + 2;
+    vector args = (*fn_iter)->get_arglist()->get_members();
+    vector::iterator arg_iter = args.begin();
+    if (arg_iter != args.end()) {
+      for ( ; arg_iter != args.end(); arg_iter++) {
+	if (!first) {
+	  f_out_ << "," << endl;
+	  for (int i = 0; i < offset; ++i) {
+	    f_out_ << " ";
+	  }
+	}
+	first = false;
+	print_type((*arg_iter)->get_type());
+	f_out_ << " " << (*arg_iter)->get_name();
+	if ((*arg_iter)->get_value() != NULL) {
+	  f_out_ << " = ";
+	  print_const_value((*arg_iter)->get_value());
+	}
+      }
+    }
+    f_out_ << ")" << endl;
+    first = true;
+    vector excepts = (*fn_iter)->get_xceptions()->get_members();
+    vector::iterator ex_iter = excepts.begin();
+    if (ex_iter != excepts.end()) {
+      f_out_ << "    throws ";
+      for ( ; ex_iter != excepts.end(); ex_iter++) {
+	if (!first) {
+	  f_out_ << ", ";
+	}
+	first = false;
+	print_type((*ex_iter)->get_type());
+      }
+      f_out_ << endl;
+    }
+    f_out_ << "
"; + print_doc(*fn_iter); + f_out_ << "
"; + } +} + +THRIFT_REGISTER_GENERATOR(html, "HTML", ""); diff --git a/compiler/cpp/src/generate/t_java_generator.cc b/compiler/cpp/src/generate/t_java_generator.cc new file mode 100644 index 00000000..3ec816fd --- /dev/null +++ b/compiler/cpp/src/generate/t_java_generator.cc @@ -0,0 +1,3008 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "platform.h" +#include "t_oop_generator.h" +using namespace std; + + +/** + * Java code generator. + * + */ +class t_java_generator : public t_oop_generator { + public: + t_java_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_oop_generator(program) + { + std::map::const_iterator iter; + + iter = parsed_options.find("beans"); + bean_style_ = (iter != parsed_options.end()); + + iter = parsed_options.find("nocamel"); + nocamel_style_ = (iter != parsed_options.end()); + + iter = parsed_options.find("hashcode"); + gen_hash_code_ = (iter != parsed_options.end()); + + out_dir_base_ = (bean_style_ ? "gen-javabean" : "gen-java"); + } + + /** + * Init and close methods + */ + + void init_generator(); + void close_generator(); + + void generate_consts(std::vector consts); + + /** + * Program-level generation functions + */ + + void generate_typedef (t_typedef* ttypedef); + void generate_enum (t_enum* tenum); + void generate_struct (t_struct* tstruct); + void generate_xception(t_struct* txception); + void generate_service (t_service* tservice); + + void print_const_value(std::ofstream& out, std::string name, t_type* type, t_const_value* value, bool in_static, bool defval=false); + std::string render_const_value(std::ofstream& out, std::string name, t_type* type, t_const_value* value); + + /** + * Service-level generation functions + */ + + void generate_java_struct(t_struct* tstruct, bool is_exception); + + void generate_java_struct_definition(std::ofstream& out, t_struct* tstruct, bool is_xception=false, bool in_class=false, bool is_result=false); + void generate_java_struct_equality(std::ofstream& out, t_struct* tstruct); + void generate_java_struct_reader(std::ofstream& out, t_struct* tstruct); + void generate_java_validator(std::ofstream& out, t_struct* tstruct); + void generate_java_struct_result_writer(std::ofstream& out, t_struct* tstruct); + void generate_java_struct_writer(std::ofstream& out, t_struct* tstruct); + void generate_java_struct_tostring(std::ofstream& out, t_struct* tstruct); + void generate_java_meta_data_map(std::ofstream& out, t_struct* tstruct); + void generate_field_value_meta_data(std::ofstream& out, t_type* type); + std::string get_java_type_string(t_type* type); + void generate_reflection_setters(std::ostringstream& out, t_type* type, std::string field_name, std::string cap_name); + void generate_reflection_getters(std::ostringstream& out, t_type* type, std::string field_name, std::string cap_name); + void generate_generic_field_getters_setters(std::ofstream& out, t_struct* tstruct); + void generate_generic_isset_method(std::ofstream& out, t_struct* tstruct); + void generate_java_bean_boilerplate(std::ofstream& out, t_struct* tstruct); + + void generate_function_helpers(t_function* tfunction); + std::string get_cap_name(std::string name); + std::string generate_isset_check(t_field* field); + std::string generate_isset_check(std::string field); + void generate_isset_set(ofstream& out, t_field* field); + + void generate_service_interface (t_service* tservice); + void generate_service_helpers (t_service* tservice); + void generate_service_client (t_service* tservice); + void generate_service_server (t_service* tservice); + void generate_process_function (t_service* tservice, t_function* tfunction); + + /** + * Serialization constructs + */ + + void generate_deserialize_field (std::ofstream& out, + t_field* tfield, + std::string prefix=""); + + void generate_deserialize_struct (std::ofstream& out, + t_struct* tstruct, + std::string prefix=""); + + void generate_deserialize_container (std::ofstream& out, + t_type* ttype, + std::string prefix=""); + + void generate_deserialize_set_element (std::ofstream& out, + t_set* tset, + std::string prefix=""); + + void generate_deserialize_map_element (std::ofstream& out, + t_map* tmap, + std::string prefix=""); + + void generate_deserialize_list_element (std::ofstream& out, + t_list* tlist, + std::string prefix=""); + + void generate_serialize_field (std::ofstream& out, + t_field* tfield, + std::string prefix=""); + + void generate_serialize_struct (std::ofstream& out, + t_struct* tstruct, + std::string prefix=""); + + void generate_serialize_container (std::ofstream& out, + t_type* ttype, + std::string prefix=""); + + void generate_serialize_map_element (std::ofstream& out, + t_map* tmap, + std::string iter, + std::string map); + + void generate_serialize_set_element (std::ofstream& out, + t_set* tmap, + std::string iter); + + void generate_serialize_list_element (std::ofstream& out, + t_list* tlist, + std::string iter); + + void generate_java_doc (std::ofstream& out, + t_doc* tdoc); + + void generate_java_doc (std::ofstream& out, + t_function* tdoc); + + void generate_deep_copy_container(std::ofstream& out, std::string source_name_p1, std::string source_name_p2, std::string result_name, t_type* type); + void generate_deep_copy_non_container(std::ofstream& out, std::string source_name, std::string dest_name, t_type* type); + + /** + * Helper rendering functions + */ + + std::string java_package(); + std::string java_type_imports(); + std::string java_thrift_imports(); + std::string type_name(t_type* ttype, bool in_container=false, bool in_init=false); + std::string base_type_name(t_base_type* tbase, bool in_container=false); + std::string declare_field(t_field* tfield, bool init=false); + std::string function_signature(t_function* tfunction, std::string prefix=""); + std::string argument_list(t_struct* tstruct); + std::string type_to_enum(t_type* ttype); + std::string get_enum_class_name(t_type* type); + + bool type_can_be_null(t_type* ttype) { + ttype = get_true_type(ttype); + + return + ttype->is_container() || + ttype->is_struct() || + ttype->is_xception() || + ttype->is_string(); + } + + std::string constant_name(std::string name); + + private: + + /** + * File streams + */ + + std::string package_name_; + std::ofstream f_service_; + std::string package_dir_; + + bool bean_style_; + bool nocamel_style_; + bool gen_hash_code_; + +}; + + +/** + * Prepares for file generation by opening up the necessary file output + * streams. + * + * @param tprogram The program to generate + */ +void t_java_generator::init_generator() { + // Make output directory + MKDIR(get_out_dir().c_str()); + package_name_ = program_->get_namespace("java"); + + string dir = package_name_; + string subdir = get_out_dir(); + string::size_type loc; + while ((loc = dir.find(".")) != string::npos) { + subdir = subdir + "/" + dir.substr(0, loc); + MKDIR(subdir.c_str()); + dir = dir.substr(loc+1); + } + if (dir.size() > 0) { + subdir = subdir + "/" + dir; + MKDIR(subdir.c_str()); + } + + package_dir_ = subdir; +} + +/** + * Packages the generated file + * + * @return String of the package, i.e. "package org.apache.thriftdemo;" + */ +string t_java_generator::java_package() { + if (!package_name_.empty()) { + return string("package ") + package_name_ + ";\n\n"; + } + return ""; +} + +/** + * Prints standard java imports + * + * @return List of imports for Java types that are used in here + */ +string t_java_generator::java_type_imports() { + string hash_builder; + if (gen_hash_code_) { + hash_builder = "import org.apache.commons.lang.builder.HashCodeBuilder;\n"; + } + + return + string() + + hash_builder + + "import java.util.List;\n" + + "import java.util.ArrayList;\n" + + "import java.util.Map;\n" + + "import java.util.HashMap;\n" + + "import java.util.Set;\n" + + "import java.util.HashSet;\n" + + "import java.util.Collections;\n\n"; +} + +/** + * Prints standard java imports + * + * @return List of imports necessary for thrift + */ +string t_java_generator::java_thrift_imports() { + return + string() + + "import org.apache.thrift.*;\n" + + "import org.apache.thrift.meta_data.*;\n" + + "import org.apache.thrift.protocol.*;\n\n"; +} + +/** + * Nothing in Java + */ +void t_java_generator::close_generator() {} + +/** + * Generates a typedef. This is not done in Java, since it does + * not support arbitrary name replacements, and it'd be a wacky waste + * of overhead to make wrapper classes. + * + * @param ttypedef The type definition + */ +void t_java_generator::generate_typedef(t_typedef* ttypedef) {} + +/** + * Enums are a class with a set of static constants. + * + * @param tenum The enumeration + */ +void t_java_generator::generate_enum(t_enum* tenum) { + // Make output file + string f_enum_name = package_dir_+"/"+(tenum->get_name())+".java"; + ofstream f_enum; + f_enum.open(f_enum_name.c_str()); + + // Comment and package it + f_enum << + autogen_comment() << + java_package() << endl; + + // Add java imports + f_enum << string() + + "import java.util.Set;\n" + + "import java.util.HashSet;\n" + + "import java.util.Collections;\n" + + "import org.apache.thrift.IntRangeSet;\n" + + "import java.util.Map;\n" + + "import java.util.HashMap;\n" << endl; + + f_enum << + "public class " << tenum->get_name() << " "; + scope_up(f_enum); + + vector constants = tenum->get_constants(); + vector::iterator c_iter; + int value = -1; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + + indent(f_enum) << + "public static final int " << (*c_iter)->get_name() << + " = " << value << ";" << endl; + } + + // Create a static Set with all valid values for this enum + f_enum << endl; + indent(f_enum) << "public static final IntRangeSet VALID_VALUES = new IntRangeSet("; + indent_up(); + bool first = true; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + // populate set + if ((*c_iter)->has_value()) { + f_enum << (first ? "" : ", ") << (*c_iter)->get_name(); + first = false; + } + } + indent_down(); + f_enum << ");" << endl; + + indent(f_enum) << "public static final Map VALUES_TO_NAMES = new HashMap() {{" << endl; + + indent_up(); + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + indent(f_enum) << "put(" << (*c_iter)->get_name() << ", \"" << (*c_iter)->get_name() <<"\");" << endl; + } + indent_down(); + + + indent(f_enum) << "}};" << endl; + + scope_down(f_enum); + + f_enum.close(); +} + +/** + * Generates a class that holds all the constants. + */ +void t_java_generator::generate_consts(std::vector consts) { + if (consts.empty()) { + return; + } + + string f_consts_name = package_dir_+"/Constants.java"; + ofstream f_consts; + f_consts.open(f_consts_name.c_str()); + + // Print header + f_consts << + autogen_comment() << + java_package() << + java_type_imports(); + + f_consts << + "public class Constants {" << endl << + endl; + indent_up(); + vector::iterator c_iter; + for (c_iter = consts.begin(); c_iter != consts.end(); ++c_iter) { + print_const_value(f_consts, + (*c_iter)->get_name(), + (*c_iter)->get_type(), + (*c_iter)->get_value(), + false); + } + indent_down(); + indent(f_consts) << + "}" << endl; + f_consts.close(); +} + + +/** + * Prints the value of a constant with the given type. Note that type checking + * is NOT performed in this function as it is always run beforehand using the + * validate_types method in main.cc + */ +void t_java_generator::print_const_value(std::ofstream& out, string name, t_type* type, t_const_value* value, bool in_static, bool defval) { + type = get_true_type(type); + + indent(out); + if (!defval) { + out << + (in_static ? "" : "public static final ") << + type_name(type) << " "; + } + if (type->is_base_type()) { + string v2 = render_const_value(out, name, type, value); + out << name << " = " << v2 << ";" << endl << endl; + } else if (type->is_enum()) { + out << name << " = " << value->get_integer() << ";" << endl << endl; + } else if (type->is_struct() || type->is_xception()) { + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + const map& val = value->get_map(); + map::const_iterator v_iter; + out << name << " = new " << type_name(type, false, true) << "();" << endl; + if (!in_static) { + indent(out) << "static {" << endl; + indent_up(); + } + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + string val = render_const_value(out, name, field_type, v_iter->second); + indent(out) << name << "."; + std::string cap_name = get_cap_name(v_iter->first->get_string()); + out << "set" << cap_name << "(" << val << ");" << endl; + } + if (!in_static) { + indent_down(); + indent(out) << "}" << endl; + } + out << endl; + } else if (type->is_map()) { + out << name << " = new " << type_name(type, false, true) << "();" << endl; + if (!in_static) { + indent(out) << "static {" << endl; + indent_up(); + } + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + string key = render_const_value(out, name, ktype, v_iter->first); + string val = render_const_value(out, name, vtype, v_iter->second); + indent(out) << name << ".put(" << key << ", " << val << ");" << endl; + } + if (!in_static) { + indent_down(); + indent(out) << "}" << endl; + } + out << endl; + } else if (type->is_list() || type->is_set()) { + out << name << " = new " << type_name(type, false, true) << "();" << endl; + if (!in_static) { + indent(out) << "static {" << endl; + indent_up(); + } + t_type* etype; + if (type->is_list()) { + etype = ((t_list*)type)->get_elem_type(); + } else { + etype = ((t_set*)type)->get_elem_type(); + } + const vector& val = value->get_list(); + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + string val = render_const_value(out, name, etype, *v_iter); + indent(out) << name << ".add(" << val << ");" << endl; + } + if (!in_static) { + indent_down(); + indent(out) << "}" << endl; + } + out << endl; + } else { + throw "compiler error: no const of type " + type->get_name(); + } +} + +string t_java_generator::render_const_value(ofstream& out, string name, t_type* type, t_const_value* value) { + type = get_true_type(type); + std::ostringstream render; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + render << '"' << get_escaped_string(value) << '"'; + break; + case t_base_type::TYPE_BOOL: + render << ((value->get_integer() > 0) ? "true" : "false"); + break; + case t_base_type::TYPE_BYTE: + render << "(byte)" << value->get_integer(); + break; + case t_base_type::TYPE_I16: + render << "(short)" << value->get_integer(); + break; + case t_base_type::TYPE_I32: + render << value->get_integer(); + break; + case t_base_type::TYPE_I64: + render << value->get_integer() << "L"; + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() == t_const_value::CV_INTEGER) { + render << "(double)" << value->get_integer(); + } else { + render << value->get_double(); + } + break; + default: + throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + render << value->get_integer(); + } else { + string t = tmp("tmp"); + print_const_value(out, t, type, value, true); + render << t; + } + + return render.str(); +} + +/** + * Generates a struct definition for a thrift data type. This is a class + * with data members, read(), write(), and an inner Isset class. + * + * @param tstruct The struct definition + */ +void t_java_generator::generate_struct(t_struct* tstruct) { + generate_java_struct(tstruct, false); +} + +/** + * Exceptions are structs, but they inherit from Exception + * + * @param tstruct The struct definition + */ +void t_java_generator::generate_xception(t_struct* txception) { + generate_java_struct(txception, true); +} + + +/** + * Java struct definition. + * + * @param tstruct The struct definition + */ +void t_java_generator::generate_java_struct(t_struct* tstruct, + bool is_exception) { + // Make output file + string f_struct_name = package_dir_+"/"+(tstruct->get_name())+".java"; + ofstream f_struct; + f_struct.open(f_struct_name.c_str()); + + f_struct << + autogen_comment() << + java_package() << + java_type_imports() << + java_thrift_imports(); + + generate_java_struct_definition(f_struct, + tstruct, + is_exception); + f_struct.close(); +} + +/** + * Java struct definition. This has various parameters, as it could be + * generated standalone or inside another class as a helper. If it + * is a helper than it is a static class. + * + * @param tstruct The struct definition + * @param is_exception Is this an exception? + * @param in_class If inside a class, needs to be static class + * @param is_result If this is a result it needs a different writer + */ +void t_java_generator::generate_java_struct_definition(ofstream &out, + t_struct* tstruct, + bool is_exception, + bool in_class, + bool is_result) { + generate_java_doc(out, tstruct); + + bool is_final = (tstruct->annotations_.find("final") != tstruct->annotations_.end()); + + indent(out) << + "public " << (is_final ? "final " : "") << + (in_class ? "static " : "") << "class " << tstruct->get_name() << " "; + + if (is_exception) { + out << "extends Exception "; + } + out << "implements TBase, java.io.Serializable, Cloneable "; + + scope_up(out); + + indent(out) << + "private static final TStruct STRUCT_DESC = new TStruct(\"" << tstruct->get_name() << "\");" << endl; + + // Members are public for -java, private for -javabean + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + indent(out) << + "private static final TField " << constant_name((*m_iter)->get_name()) << + "_FIELD_DESC = new TField(\"" << (*m_iter)->get_name() << "\", " << + type_to_enum((*m_iter)->get_type()) << ", " << + "(short)" << (*m_iter)->get_key() << ");" << endl; + } + + out << endl; + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + if (bean_style_) { + indent(out) << "private "; + } else { + generate_java_doc(out, *m_iter); + indent(out) << "public "; + } + out << declare_field(*m_iter, false) << endl; + + indent(out) << "public static final int " << upcase_string((*m_iter)->get_name()) << " = " << (*m_iter)->get_key() << ";" << endl; + } + + // Inner Isset class + if (members.size() > 0) { + out << + endl << + indent() << "private final Isset __isset = new Isset();" << endl << + indent() << "private static final class Isset implements java.io.Serializable {" << endl; + indent_up(); + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + if (!type_can_be_null((*m_iter)->get_type())){ + indent(out) << + "public boolean " << (*m_iter)->get_name() << " = false;" << endl; + } + } + indent_down(); + out << + indent() << "}" << endl << + endl; + } + + generate_java_meta_data_map(out, tstruct); + + // Static initializer to populate global class to struct metadata map + indent(out) << "static {" << endl; + indent_up(); + indent(out) << "FieldMetaData.addStructMetaDataMap(" << type_name(tstruct) << ".class, metaDataMap);" << endl; + indent_down(); + indent(out) << "}" << endl << endl; + + // Default constructor + indent(out) << + "public " << tstruct->get_name() << "() {" << endl; + indent_up(); + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + t_type* t = get_true_type((*m_iter)->get_type()); + if ((*m_iter)->get_value() != NULL) { + print_const_value(out, "this." + (*m_iter)->get_name(), t, (*m_iter)->get_value(), true, true); + } + } + indent_down(); + indent(out) << "}" << endl << endl; + + + if (!members.empty()) { + // Full constructor for all fields + indent(out) << + "public " << tstruct->get_name() << "(" << endl; + indent_up(); + for (m_iter = members.begin(); m_iter != members.end(); ) { + indent(out) << type_name((*m_iter)->get_type()) << " " << + (*m_iter)->get_name(); + ++m_iter; + if (m_iter != members.end()) { + out << "," << endl; + } + } + out << ")" << endl; + indent_down(); + indent(out) << "{" << endl; + indent_up(); + indent(out) << "this();" << endl; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + indent(out) << "this." << (*m_iter)->get_name() << " = " << + (*m_iter)->get_name() << ";" << endl; + generate_isset_set(out, (*m_iter)); + } + indent_down(); + indent(out) << "}" << endl << endl; + } + + // copy constructor + indent(out) << "/**" << endl; + indent(out) << " * Performs a deep copy on other." << endl; + indent(out) << " */" << endl; + indent(out) << "public " << tstruct->get_name() << "(" << tstruct->get_name() << " other) {" << endl; + indent_up(); + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + t_field* field = (*m_iter); + std::string field_name = field->get_name(); + t_type* type = field->get_type(); + bool can_be_null = type_can_be_null(type); + + if (!can_be_null) { + indent(out) << "__isset." << field_name << " = other.__isset." << field_name << ";" << endl; + } + + if (can_be_null) { + indent(out) << "if (other." << generate_isset_check(field) << ") {" << endl; + indent_up(); + } + + if (type->is_container()) { + generate_deep_copy_container(out, "other", field_name, "__this__" + field_name, type); + indent(out) << "this." << field_name << " = __this__" << field_name << ";" << endl; + } else { + indent(out) << "this." << field_name << " = "; + generate_deep_copy_non_container(out, "other." + field_name, field_name, type); + out << ";" << endl; + } + + if (can_be_null) { + indent_down(); + indent(out) << "}" << endl; + } + } + + indent_down(); + indent(out) << "}" << endl << endl; + + // clone method, so that you can deep copy an object when you don't know its class. + indent(out) << "@Override" << endl; + indent(out) << "public " << tstruct->get_name() << " clone() {" << endl; + indent(out) << " return new " << tstruct->get_name() << "(this);" << endl; + indent(out) << "}" << endl << endl; + + generate_java_bean_boilerplate(out, tstruct); + generate_generic_field_getters_setters(out, tstruct); + generate_generic_isset_method(out, tstruct); + + generate_java_struct_equality(out, tstruct); + + generate_java_struct_reader(out, tstruct); + if (is_result) { + generate_java_struct_result_writer(out, tstruct); + } else { + generate_java_struct_writer(out, tstruct); + } + generate_java_struct_tostring(out, tstruct); + generate_java_validator(out, tstruct); + scope_down(out); + out << endl; +} + +/** + * Generates equals methods and a hashCode method for a structure. + * + * @param tstruct The struct definition + */ +void t_java_generator::generate_java_struct_equality(ofstream& out, + t_struct* tstruct) { + out << indent() << "@Override" << endl << + indent() << "public boolean equals(Object that) {" << endl; + indent_up(); + out << + indent() << "if (that == null)" << endl << + indent() << " return false;" << endl << + indent() << "if (that instanceof " << tstruct->get_name() << ")" << endl << + indent() << " return this.equals((" << tstruct->get_name() << ")that);" << endl << + indent() << "return false;" << endl; + scope_down(out); + out << endl; + + out << + indent() << "public boolean equals(" << tstruct->get_name() << " that) {" << endl; + indent_up(); + out << + indent() << "if (that == null)" << endl << + indent() << " return false;" << endl; + + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + out << endl; + + t_type* t = get_true_type((*m_iter)->get_type()); + // Most existing Thrift code does not use isset or optional/required, + // so we treat "default" fields as required. + bool is_optional = (*m_iter)->get_req() == t_field::T_OPTIONAL; + bool can_be_null = type_can_be_null(t); + string name = (*m_iter)->get_name(); + + string this_present = "true"; + string that_present = "true"; + string unequal; + + if (is_optional || can_be_null) { + this_present += " && this." + generate_isset_check(*m_iter); + that_present += " && that." + generate_isset_check(*m_iter); + } + + out << + indent() << "boolean this_present_" << name << " = " + << this_present << ";" << endl << + indent() << "boolean that_present_" << name << " = " + << that_present << ";" << endl << + indent() << "if (" << "this_present_" << name + << " || that_present_" << name << ") {" << endl; + indent_up(); + out << + indent() << "if (!(" << "this_present_" << name + << " && that_present_" << name << "))" << endl << + indent() << " return false;" << endl; + + if (t->is_base_type() && ((t_base_type*)t)->is_binary()) { + unequal = "!java.util.Arrays.equals(this." + name + ", that." + name + ")"; + } else if (can_be_null) { + unequal = "!this." + name + ".equals(that." + name + ")"; + } else { + unequal = "this." + name + " != that." + name; + } + + out << + indent() << "if (" << unequal << ")" << endl << + indent() << " return false;" << endl; + + scope_down(out); + } + out << endl; + indent(out) << "return true;" << endl; + scope_down(out); + out << endl; + + out << indent() << "@Override" << endl << + indent() << "public int hashCode() {" << endl; + indent_up(); + if (gen_hash_code_) { + indent(out) << "HashCodeBuilder builder = new HashCodeBuilder();" << endl; + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + out << endl; + + t_type* t = get_true_type((*m_iter)->get_type()); + bool is_optional = (*m_iter)->get_req() == t_field::T_OPTIONAL; + bool can_be_null = type_can_be_null(t); + string name = (*m_iter)->get_name(); + + string present = "true"; + + if (is_optional || can_be_null) { + present += " && (" + generate_isset_check(*m_iter) + ")"; + } + + out << + indent() << "boolean present_" << name << " = " + << present << ";" << endl << + indent() << "builder.append(present_" << name << ");" << endl << + indent() << "if (present_" << name << ")" << endl << + indent() << " builder.append(" << name << ");" << endl; + } + + out << endl; + indent(out) << "return builder.toHashCode();" << endl; + } else { + indent(out) << "return 0;" << endl; + } + indent_down(); + indent(out) << "}" << endl << endl; +} + +/** + * Generates a function to read all the fields of the struct. + * + * @param tstruct The struct definition + */ +void t_java_generator::generate_java_struct_reader(ofstream& out, + t_struct* tstruct) { + out << + indent() << "public void read(TProtocol iprot) throws TException {" << endl; + indent_up(); + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + // Declare stack tmp variables and read struct header + out << + indent() << "TField field;" << endl << + indent() << "iprot.readStructBegin();" << endl; + + // Loop over reading in fields + indent(out) << + "while (true)" << endl; + scope_up(out); + + // Read beginning field marker + indent(out) << + "field = iprot.readFieldBegin();" << endl; + + // Check for field STOP marker and break + indent(out) << + "if (field.type == TType.STOP) { " << endl; + indent_up(); + indent(out) << + "break;" << endl; + indent_down(); + indent(out) << + "}" << endl; + + // Switch statement on the field we are reading + indent(out) << + "switch (field.id)" << endl; + + scope_up(out); + + // Generate deserialization code for known cases + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + indent(out) << + "case " << upcase_string((*f_iter)->get_name()) << ":" << endl; + indent_up(); + indent(out) << + "if (field.type == " << type_to_enum((*f_iter)->get_type()) << ") {" << endl; + indent_up(); + + generate_deserialize_field(out, *f_iter, "this."); + generate_isset_set(out, *f_iter); + indent_down(); + out << + indent() << "} else { " << endl << + indent() << " TProtocolUtil.skip(iprot, field.type);" << endl << + indent() << "}" << endl << + indent() << "break;" << endl; + indent_down(); + } + + // In the default case we skip the field + out << + indent() << "default:" << endl << + indent() << " TProtocolUtil.skip(iprot, field.type);" << endl << + indent() << " break;" << endl; + + scope_down(out); + + // Read field end marker + indent(out) << + "iprot.readFieldEnd();" << endl; + + scope_down(out); + + out << + indent() << "iprot.readStructEnd();" << endl << endl; + + // in non-beans style, check for required fields of primitive type + // (which can be checked here but not in the general validate method) + if (!bean_style_){ + out << endl << indent() << "// check for required fields of primitive type, which can't be checked in the validate method" << endl; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_req() == t_field::T_REQUIRED && !type_can_be_null((*f_iter)->get_type())) { + out << + indent() << "if (!__isset." << (*f_iter)->get_name() << ") {" << endl << + indent() << " throw new TProtocolException(\"Required field '" << (*f_iter)->get_name() << "' was not found in serialized data! Struct: \" + toString());" << endl << + indent() << "}" << endl; + } + } + } + + // performs various checks (e.g. check that all required fields are set) + indent(out) << "validate();" << endl; + + indent_down(); + out << + indent() << "}" << endl << + endl; +} + +// generates java method to perform various checks +// (e.g. check that all required fields are set) +void t_java_generator::generate_java_validator(ofstream& out, + t_struct* tstruct){ + indent(out) << "public void validate() throws TException {" << endl; + indent_up(); + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + out << indent() << "// check for required fields" << endl; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_req() == t_field::T_REQUIRED) { + if (bean_style_) { + out << + indent() << "if (!" << generate_isset_check(*f_iter) << ") {" << endl << + indent() << " throw new TProtocolException(\"Required field '" << (*f_iter)->get_name() << "' is unset! Struct:\" + toString());" << endl << + indent() << "}" << endl << endl; + } else{ + if (type_can_be_null((*f_iter)->get_type())) { + indent(out) << "if (" << (*f_iter)->get_name() << " == null) {" << endl; + indent(out) << " throw new TProtocolException(\"Required field '" << (*f_iter)->get_name() << "' was not present! Struct: \" + toString());" << endl; + indent(out) << "}" << endl; + } else { + indent(out) << "// alas, we cannot check '" << (*f_iter)->get_name() << "' because it's a primitive and you chose the non-beans generator." << endl; + } + } + } + } + + // check that fields of type enum have valid values + out << indent() << "// check that fields of type enum have valid values" << endl; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + t_field* field = (*f_iter); + t_type* type = field->get_type(); + // if field is an enum, check that its value is valid + if (type->is_enum()){ + indent(out) << "if (" << generate_isset_check(field) << " && !" << get_enum_class_name(type) << ".VALID_VALUES.contains(" << field->get_name() << ")){" << endl; + indent_up(); + indent(out) << "throw new TProtocolException(\"The field '" << field->get_name() << "' has been assigned the invalid value \" + " << field->get_name() << ");" << endl; + indent_down(); + indent(out) << "}" << endl; + } + } + + indent_down(); + indent(out) << "}" << endl << endl; +} + +/** + * Generates a function to write all the fields of the struct + * + * @param tstruct The struct definition + */ +void t_java_generator::generate_java_struct_writer(ofstream& out, + t_struct* tstruct) { + out << + indent() << "public void write(TProtocol oprot) throws TException {" << endl; + indent_up(); + + string name = tstruct->get_name(); + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + // performs various checks (e.g. check that all required fields are set) + indent(out) << "validate();" << endl << endl; + + indent(out) << "oprot.writeStructBegin(STRUCT_DESC);" << endl; + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + bool null_allowed = type_can_be_null((*f_iter)->get_type()); + if (null_allowed) { + out << + indent() << "if (this." << (*f_iter)->get_name() << " != null) {" << endl; + indent_up(); + } + bool optional = bean_style_ && (*f_iter)->get_req() == t_field::T_OPTIONAL; + if (optional) { + indent(out) << "if (" << generate_isset_check((*f_iter)) << ") {" << endl; + indent_up(); + } + + indent(out) << "oprot.writeFieldBegin(" << constant_name((*f_iter)->get_name()) << "_FIELD_DESC);" << endl; + + // Write field contents + generate_serialize_field(out, *f_iter, "this."); + + // Write field closer + indent(out) << + "oprot.writeFieldEnd();" << endl; + + if (optional) { + indent_down(); + indent(out) << "}" << endl; + } + if (null_allowed) { + indent_down(); + indent(out) << "}" << endl; + } + } + // Write the struct map + out << + indent() << "oprot.writeFieldStop();" << endl << + indent() << "oprot.writeStructEnd();" << endl; + + indent_down(); + out << + indent() << "}" << endl << + endl; +} + +/** + * Generates a function to write all the fields of the struct, + * which is a function result. These fields are only written + * if they are set in the Isset array, and only one of them + * can be set at a time. + * + * @param tstruct The struct definition + */ +void t_java_generator::generate_java_struct_result_writer(ofstream& out, + t_struct* tstruct) { + out << + indent() << "public void write(TProtocol oprot) throws TException {" << endl; + indent_up(); + + string name = tstruct->get_name(); + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + indent(out) << "oprot.writeStructBegin(STRUCT_DESC);" << endl; + + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + out << + endl << + indent() << "if "; + } else { + out << " else if "; + } + + out << "(this." << generate_isset_check(*f_iter) << ") {" << endl; + + indent_up(); + + indent(out) << "oprot.writeFieldBegin(" << constant_name((*f_iter)->get_name()) << "_FIELD_DESC);" << endl; + + // Write field contents + generate_serialize_field(out, *f_iter, "this."); + + // Write field closer + indent(out) << + "oprot.writeFieldEnd();" << endl; + + indent_down(); + indent(out) << "}"; + } + // Write the struct map + out << + endl << + indent() << "oprot.writeFieldStop();" << endl << + indent() << "oprot.writeStructEnd();" << endl; + + indent_down(); + out << + indent() << "}" << endl << + endl; +} + +void t_java_generator::generate_reflection_getters(ostringstream& out, t_type* type, string field_name, string cap_name) { + indent(out) << "case " << upcase_string(field_name) << ":" << endl; + indent_up(); + + if (type->is_base_type() && !type->is_string()) { + t_base_type* base_type = (t_base_type*)type; + + indent(out) << "return new " << type_name(type, true, false) << "(" << (base_type->is_bool() ? "is" : "get") << cap_name << "());" << endl << endl; + } else { + indent(out) << "return get" << cap_name << "();" << endl << endl; + } + + indent_down(); +} + +void t_java_generator::generate_reflection_setters(ostringstream& out, t_type* type, string field_name, string cap_name) { + indent(out) << "case " << upcase_string(field_name) << ":" << endl; + indent_up(); + indent(out) << "if (value == null) {" << endl; + indent(out) << " unset" << get_cap_name(field_name) << "();" << endl; + indent(out) << "} else {" << endl; + indent(out) << " set" << cap_name << "((" << type_name(type, true, false) << ")value);" << endl; + indent(out) << "}" << endl; + indent(out) << "break;" << endl << endl; + + indent_down(); +} + +void t_java_generator::generate_generic_field_getters_setters(std::ofstream& out, t_struct* tstruct) { + + std::ostringstream getter_stream; + std::ostringstream setter_stream; + + // build up the bodies of both the getter and setter at once + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + t_field* field = *f_iter; + t_type* type = get_true_type(field->get_type()); + std::string field_name = field->get_name(); + std::string cap_name = get_cap_name(field_name); + + indent_up(); + generate_reflection_setters(setter_stream, type, field_name, cap_name); + generate_reflection_getters(getter_stream, type, field_name, cap_name); + indent_down(); + } + + + // create the setter + indent(out) << "public void setFieldValue(int fieldID, Object value) {" << endl; + indent_up(); + + indent(out) << "switch (fieldID) {" << endl; + + out << setter_stream.str(); + + indent(out) << "default:" << endl; + indent(out) << " throw new IllegalArgumentException(\"Field \" + fieldID + \" doesn't exist!\");" << endl; + + indent(out) << "}" << endl; + + indent_down(); + indent(out) << "}" << endl << endl; + + // create the getter + indent(out) << "public Object getFieldValue(int fieldID) {" << endl; + indent_up(); + + indent(out) << "switch (fieldID) {" << endl; + + out << getter_stream.str(); + + indent(out) << "default:" << endl; + indent(out) << " throw new IllegalArgumentException(\"Field \" + fieldID + \" doesn't exist!\");" << endl; + + indent(out) << "}" << endl; + + indent_down(); + + indent(out) << "}" << endl << endl; +} + +// Creates a generic isSet method that takes the field number as argument +void t_java_generator::generate_generic_isset_method(std::ofstream& out, t_struct* tstruct){ + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + // create the isSet method + indent(out) << "// Returns true if field corresponding to fieldID is set (has been asigned a value) and false otherwise" << endl; + indent(out) << "public boolean isSet(int fieldID) {" << endl; + indent_up(); + indent(out) << "switch (fieldID) {" << endl; + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + t_field* field = *f_iter; + indent(out) << "case " << upcase_string(field->get_name()) << ":" << endl; + indent_up(); + indent(out) << "return " << generate_isset_check(field) << ";" << endl; + indent_down(); + } + + indent(out) << "default:" << endl; + indent(out) << " throw new IllegalArgumentException(\"Field \" + fieldID + \" doesn't exist!\");" << endl; + + indent(out) << "}" << endl; + + indent_down(); + indent(out) << "}" << endl << endl; +} + +/** + * Generates a set of Java Bean boilerplate functions (setters, getters, etc.) + * for the given struct. + * + * @param tstruct The struct definition + */ +void t_java_generator::generate_java_bean_boilerplate(ofstream& out, + t_struct* tstruct) { + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + t_field* field = *f_iter; + t_type* type = get_true_type(field->get_type()); + std::string field_name = field->get_name(); + std::string cap_name = get_cap_name(field_name); + + if (type->is_container()) { + // Method to return the size of the collection + indent(out) << "public int get" << cap_name; + out << get_cap_name("size() {") << endl; + + indent_up(); + indent(out) << "return (this." << field_name << " == null) ? 0 : " << + "this." << field_name << ".size();" << endl; + indent_down(); + indent(out) << "}" << endl << endl; + } + + if (type->is_set() || type->is_list()) { + + t_type* element_type; + if (type->is_set()) { + element_type = ((t_set*)type)->get_elem_type(); + } else { + element_type = ((t_list*)type)->get_elem_type(); + } + + // Iterator getter for sets and lists + indent(out) << "public java.util.Iterator<" << + type_name(element_type, true, false) << "> get" << cap_name; + out << get_cap_name("iterator() {") << endl; + + indent_up(); + indent(out) << "return (this." << field_name << " == null) ? null : " << + "this." << field_name << ".iterator();" << endl; + indent_down(); + indent(out) << "}" << endl << endl; + + // Add to set or list, create if the set/list is null + indent(out); + out << "public void add" << get_cap_name("to"); + out << cap_name << "(" << type_name(element_type) << " elem) {" << endl; + + indent_up(); + indent(out) << "if (this." << field_name << " == null) {" << endl; + indent_up(); + indent(out) << "this." << field_name << " = new " << type_name(type, false, true) << + "();" << endl; + indent_down(); + indent(out) << "}" << endl; + indent(out) << "this." << field_name << ".add(elem);" << endl; + indent_down(); + indent(out) << "}" << endl << endl; + + } else if (type->is_map()) { + // Put to map + t_type* key_type = ((t_map*)type)->get_key_type(); + t_type* val_type = ((t_map*)type)->get_val_type(); + + indent(out); + out << "public void put" << get_cap_name("to"); + out << cap_name << "(" << type_name(key_type) << " key, " + << type_name(val_type) << " val) {" << endl; + + indent_up(); + indent(out) << "if (this." << field_name << " == null) {" << endl; + indent_up(); + indent(out) << "this." << field_name << " = new " << + type_name(type, false, true) << "();" << endl; + indent_down(); + indent(out) << "}" << endl; + indent(out) << "this." << field_name << ".put(key, val);" << endl; + indent_down(); + indent(out) << "}" << endl << endl; + } + + // Simple getter + generate_java_doc(out, field); + indent(out) << "public " << type_name(type); + if (type->is_base_type() && + ((t_base_type*)type)->get_base() == t_base_type::TYPE_BOOL) { + out << " is"; + } else { + out << " get"; + } + out << cap_name << "() {" << endl; + indent_up(); + indent(out) << "return this." << field_name << ";" << endl; + indent_down(); + indent(out) << "}" << endl << endl; + + // Simple setter + generate_java_doc(out, field); + indent(out) << "public void set" << cap_name << "(" << type_name(type) << + " " << field_name << ") {" << endl; + indent_up(); + indent(out) << "this." << field_name << " = " << field_name << ";" << + endl; + generate_isset_set(out, field); + + indent_down(); + indent(out) << "}" << endl << endl; + + // Unsetter + indent(out) << "public void unset" << cap_name << "() {" << endl; + indent_up(); + if (type_can_be_null(type)) { + indent(out) << "this." << field_name << " = null;" << endl; + } else { + indent(out) << "this.__isset." << field_name << " = false;" << endl; + } + indent_down(); + indent(out) << "}" << endl << endl; + + // isSet method + indent(out) << "// Returns true if field " << field_name << " is set (has been asigned a value) and false otherwise" << endl; + indent(out) << "public boolean is" << get_cap_name("set") << cap_name << "() {" << endl; + indent_up(); + if (type_can_be_null(type)) { + indent(out) << "return this." << field_name << " != null;" << endl; + } else { + indent(out) << "return this.__isset." << field_name << ";" << endl; + } + indent_down(); + indent(out) << "}" << endl << endl; + + if(!bean_style_) { + indent(out) << "public void set" << cap_name << get_cap_name("isSet") << "(boolean value) {" << endl; + indent_up(); + if (type_can_be_null(type)) { + indent(out) << "if (!value) {" << endl; + indent(out) << " this." << field_name << " = null;" << endl; + indent(out) << "}" << endl; + } else { + indent(out) << "this.__isset." << field_name << " = value;" << endl; + } + indent_down(); + indent(out) << "}" << endl << endl; + } + } +} + +/** + * Generates a toString() method for the given struct + * + * @param tstruct The struct definition + */ +void t_java_generator::generate_java_struct_tostring(ofstream& out, + t_struct* tstruct) { + out << indent() << "@Override" << endl << + indent() << "public String toString() {" << endl; + indent_up(); + + out << + indent() << "StringBuilder sb = new StringBuilder(\"" << tstruct->get_name() << "(\");" << endl; + out << indent() << "boolean first = true;" << endl << endl; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + bool could_be_unset = (*f_iter)->get_req() == t_field::T_OPTIONAL; + if(could_be_unset) { + indent(out) << "if (" << generate_isset_check(*f_iter) << ") {" << endl; + indent_up(); + } + + t_field* field = (*f_iter); + + if (!first) { + indent(out) << "if (!first) sb.append(\", \");" << endl; + } + indent(out) << "sb.append(\"" << (*f_iter)->get_name() << ":\");" << endl; + bool can_be_null = type_can_be_null(field->get_type()); + if (can_be_null) { + indent(out) << "if (this." << (*f_iter)->get_name() << " == null) {" << endl; + indent(out) << " sb.append(\"null\");" << endl; + indent(out) << "} else {" << endl; + indent_up(); + } + + if (field->get_type()->is_base_type() && ((t_base_type*)(field->get_type()))->is_binary()) { + indent(out) << " int __" << field->get_name() << "_size = Math.min(this." << field->get_name() << ".length, 128);" << endl; + indent(out) << " for (int i = 0; i < __" << field->get_name() << "_size; i++) {" << endl; + indent(out) << " if (i != 0) sb.append(\" \");" << endl; + indent(out) << " sb.append(Integer.toHexString(this." << field->get_name() << "[i]).length() > 1 ? Integer.toHexString(this." << field->get_name() << "[i]).substring(Integer.toHexString(this." << field->get_name() << "[i]).length() - 2).toUpperCase() : \"0\" + Integer.toHexString(this." << field->get_name() << "[i]).toUpperCase());" <get_name() << ".length > 128) sb.append(\" ...\");" << endl; + } else if(field->get_type()->is_enum()) { + indent(out) << "String " << field->get_name() << "_name = " << get_enum_class_name(field->get_type()) << ".VALUES_TO_NAMES.get(this." << (*f_iter)->get_name() << ");"<< endl; + indent(out) << "if (" << field->get_name() << "_name != null) {" << endl; + indent(out) << " sb.append(" << field->get_name() << "_name);" << endl; + indent(out) << " sb.append(\" (\");" << endl; + indent(out) << "}" << endl; + indent(out) << "sb.append(this." << field->get_name() << ");" << endl; + indent(out) << "if (" << field->get_name() << "_name != null) {" << endl; + indent(out) << " sb.append(\")\");" << endl; + indent(out) << "}" << endl; + } else { + indent(out) << "sb.append(this." << (*f_iter)->get_name() << ");" << endl; + } + + if (can_be_null) { + indent_down(); + indent(out) << "}" << endl; + } + indent(out) << "first = false;" << endl; + + if(could_be_unset) { + indent_down(); + indent(out) << "}" << endl; + } + first = false; + } + out << + indent() << "sb.append(\")\");" << endl << + indent() << "return sb.toString();" << endl; + + indent_down(); + indent(out) << "}" << endl << + endl; +} + +/** + * Generates a static map with meta data to store information such as fieldID to + * fieldName mapping + * + * @param tstruct The struct definition + */ +void t_java_generator::generate_java_meta_data_map(ofstream& out, + t_struct* tstruct) { + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + // Static Map with fieldID -> FieldMetaData mappings + indent(out) << "public static final Map metaDataMap = Collections.unmodifiableMap(new HashMap() {{" << endl; + + // Populate map + indent_up(); + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + t_field* field = *f_iter; + std::string field_name = field->get_name(); + indent(out) << "put(" << upcase_string(field_name) << ", new FieldMetaData(\"" << field_name << "\", "; + + // Set field requirement type (required, optional, etc.) + if (field->get_req() == t_field::T_REQUIRED) { + out << "TFieldRequirementType.REQUIRED, "; + } else if (field->get_req() == t_field::T_OPTIONAL) { + out << "TFieldRequirementType.OPTIONAL, "; + } else { + out << "TFieldRequirementType.DEFAULT, "; + } + + // Create value meta data + generate_field_value_meta_data(out, field->get_type()); + out << "));" << endl; + } + indent_down(); + indent(out) << "}});" << endl << endl; +} + +/** + * Returns a string with the java representation of the given thrift type + * (e.g. for the type struct it returns "TType.STRUCT") + */ +std::string t_java_generator::get_java_type_string(t_type* type) { + if (type->is_list()){ + return "TType.LIST"; + } else if (type->is_map()) { + return "TType.MAP"; + } else if (type->is_set()) { + return "TType.SET"; + } else if (type->is_struct() || type->is_xception()) { + return "TType.STRUCT"; + } else if (type->is_enum()) { + return "TType.I32"; + } else if (type->is_typedef()) { + return get_java_type_string(((t_typedef*)type)->get_type()); + } else if (type->is_base_type()) { + switch (((t_base_type*)type)->get_base()) { + case t_base_type::TYPE_VOID : return "TType.VOID"; break; + case t_base_type::TYPE_STRING : return "TType.STRING"; break; + case t_base_type::TYPE_BOOL : return "TType.BOOL"; break; + case t_base_type::TYPE_BYTE : return "TType.BYTE"; break; + case t_base_type::TYPE_I16 : return "TType.I16"; break; + case t_base_type::TYPE_I32 : return "TType.I32"; break; + case t_base_type::TYPE_I64 : return "TType.I64"; break; + case t_base_type::TYPE_DOUBLE : return "TType.DOUBLE"; break; + default : throw std::runtime_error("Unknown thrift type \"" + type->get_name() + "\" passed to t_java_generator::get_java_type_string!"); break; // This should never happen! + } + } else { + throw std::runtime_error("Unknown thrift type \"" + type->get_name() + "\" passed to t_java_generator::get_java_type_string!"); // This should never happen! + } +} + +void t_java_generator::generate_field_value_meta_data(std::ofstream& out, t_type* type){ + out << endl; + indent_up(); + indent_up(); + if (type->is_struct()){ + indent(out) << "new StructMetaData(TType.STRUCT, " << type_name(type) << ".class"; + } else if (type->is_container()){ + if (type->is_list()){ + indent(out) << "new ListMetaData(TType.LIST, "; + t_type* elem_type = ((t_list*)type)->get_elem_type(); + generate_field_value_meta_data(out, elem_type); + } else if (type->is_set()){ + indent(out) << "new SetMetaData(TType.SET, "; + t_type* elem_type = ((t_list*)type)->get_elem_type(); + generate_field_value_meta_data(out, elem_type); + } else{ // map + indent(out) << "new MapMetaData(TType.MAP, "; + t_type* key_type = ((t_map*)type)->get_key_type(); + t_type* val_type = ((t_map*)type)->get_val_type(); + generate_field_value_meta_data(out, key_type); + out << ", "; + generate_field_value_meta_data(out, val_type); + } + } else { + indent(out) << "new FieldValueMetaData(" << get_java_type_string(type); + } + out << ")"; + indent_down(); + indent_down(); +} + + +/** + * Generates a thrift service. In C++, this comprises an entirely separate + * header and source file. The header file defines the methods and includes + * the data types defined in the main header file, and the implementation + * file contains implementations of the basic printer and default interfaces. + * + * @param tservice The service definition + */ +void t_java_generator::generate_service(t_service* tservice) { + // Make output file + string f_service_name = package_dir_+"/"+service_name_+".java"; + f_service_.open(f_service_name.c_str()); + + f_service_ << + autogen_comment() << + java_package() << + java_type_imports() << + java_thrift_imports(); + + f_service_ << + "public class " << service_name_ << " {" << endl << + endl; + indent_up(); + + // Generate the three main parts of the service + generate_service_interface(tservice); + generate_service_client(tservice); + generate_service_server(tservice); + generate_service_helpers(tservice); + + indent_down(); + f_service_ << + "}" << endl; + f_service_.close(); +} + +/** + * Generates a service interface definition. + * + * @param tservice The service to generate a header definition for + */ +void t_java_generator::generate_service_interface(t_service* tservice) { + string extends = ""; + string extends_iface = ""; + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + extends_iface = " extends " + extends + ".Iface"; + } + + generate_java_doc(f_service_, tservice); + f_service_ << indent() << "public interface Iface" << extends_iface << + " {" << endl << endl; + indent_up(); + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + generate_java_doc(f_service_, *f_iter); + indent(f_service_) << "public " << function_signature(*f_iter) << ";" << + endl << endl; + } + indent_down(); + f_service_ << + indent() << "}" << endl << + endl; +} + +/** + * Generates structs for all the service args and return types + * + * @param tservice The service + */ +void t_java_generator::generate_service_helpers(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* ts = (*f_iter)->get_arglist(); + generate_java_struct_definition(f_service_, ts, false, true); + generate_function_helpers(*f_iter); + } +} + +/** + * Generates a service client definition. + * + * @param tservice The service to generate a server for. + */ +void t_java_generator::generate_service_client(t_service* tservice) { + string extends = ""; + string extends_client = ""; + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + extends_client = " extends " + extends + ".Client"; + } + + indent(f_service_) << + "public static class Client" << extends_client << " implements Iface {" << endl; + indent_up(); + + indent(f_service_) << + "public Client(TProtocol prot)" << endl; + scope_up(f_service_); + indent(f_service_) << + "this(prot, prot);" << endl; + scope_down(f_service_); + f_service_ << endl; + + indent(f_service_) << + "public Client(TProtocol iprot, TProtocol oprot)" << endl; + scope_up(f_service_); + if (extends.empty()) { + f_service_ << + indent() << "iprot_ = iprot;" << endl << + indent() << "oprot_ = oprot;" << endl; + } else { + f_service_ << + indent() << "super(iprot, oprot);" << endl; + } + scope_down(f_service_); + f_service_ << endl; + + if (extends.empty()) { + f_service_ << + indent() << "protected TProtocol iprot_;" << endl << + indent() << "protected TProtocol oprot_;" << endl << + endl << + indent() << "protected int seqid_;" << endl << + endl; + + indent(f_service_) << + "public TProtocol getInputProtocol()" << endl; + scope_up(f_service_); + indent(f_service_) << + "return this.iprot_;" << endl; + scope_down(f_service_); + f_service_ << endl; + + indent(f_service_) << + "public TProtocol getOutputProtocol()" << endl; + scope_up(f_service_); + indent(f_service_) << + "return this.oprot_;" << endl; + scope_down(f_service_); + f_service_ << endl; + + } + + // Generate client method implementations + vector functions = tservice->get_functions(); + vector::const_iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string funname = (*f_iter)->get_name(); + + // Open function + indent(f_service_) << + "public " << function_signature(*f_iter) << endl; + scope_up(f_service_); + indent(f_service_) << + "send_" << funname << "("; + + // Get the struct of function call params + t_struct* arg_struct = (*f_iter)->get_arglist(); + + // Declare the function arguments + const vector& fields = arg_struct->get_members(); + vector::const_iterator fld_iter; + bool first = true; + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << (*fld_iter)->get_name(); + } + f_service_ << ");" << endl; + + if (!(*f_iter)->is_oneway()) { + f_service_ << indent(); + if (!(*f_iter)->get_returntype()->is_void()) { + f_service_ << "return "; + } + f_service_ << + "recv_" << funname << "();" << endl; + } + scope_down(f_service_); + f_service_ << endl; + + t_function send_function(g_type_void, + string("send_") + (*f_iter)->get_name(), + (*f_iter)->get_arglist()); + + string argsname = (*f_iter)->get_name() + "_args"; + + // Open function + indent(f_service_) << + "public " << function_signature(&send_function) << endl; + scope_up(f_service_); + + // Serialize the request + f_service_ << + indent() << "oprot_.writeMessageBegin(new TMessage(\"" << funname << "\", TMessageType.CALL, seqid_));" << endl << + indent() << argsname << " args = new " << argsname << "();" << endl; + + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + f_service_ << + indent() << "args." << (*fld_iter)->get_name() << " = " << (*fld_iter)->get_name() << ";" << endl; + } + + f_service_ << + indent() << "args.write(oprot_);" << endl << + indent() << "oprot_.writeMessageEnd();" << endl << + indent() << "oprot_.getTransport().flush();" << endl; + + scope_down(f_service_); + f_service_ << endl; + + if (!(*f_iter)->is_oneway()) { + string resultname = (*f_iter)->get_name() + "_result"; + + t_struct noargs(program_); + t_function recv_function((*f_iter)->get_returntype(), + string("recv_") + (*f_iter)->get_name(), + &noargs, + (*f_iter)->get_xceptions()); + // Open function + indent(f_service_) << + "public " << function_signature(&recv_function) << endl; + scope_up(f_service_); + + // TODO(mcslee): Message validation here, was the seqid etc ok? + + f_service_ << + indent() << "TMessage msg = iprot_.readMessageBegin();" << endl << + indent() << "if (msg.type == TMessageType.EXCEPTION) {" << endl << + indent() << " TApplicationException x = TApplicationException.read(iprot_);" << endl << + indent() << " iprot_.readMessageEnd();" << endl << + indent() << " throw x;" << endl << + indent() << "}" << endl << + indent() << resultname << " result = new " << resultname << "();" << endl << + indent() << "result.read(iprot_);" << endl << + indent() << "iprot_.readMessageEnd();" << endl; + + // Careful, only return _result if not a void function + if (!(*f_iter)->get_returntype()->is_void()) { + f_service_ << + indent() << "if (result." << generate_isset_check("success") << ") {" << endl << + indent() << " return result.success;" << endl << + indent() << "}" << endl; + } + + t_struct* xs = (*f_iter)->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << + indent() << "if (result." << (*x_iter)->get_name() << " != null) {" << endl << + indent() << " throw result." << (*x_iter)->get_name() << ";" << endl << + indent() << "}" << endl; + } + + // If you get here it's an exception, unless a void function + if ((*f_iter)->get_returntype()->is_void()) { + indent(f_service_) << + "return;" << endl; + } else { + f_service_ << + indent() << "throw new TApplicationException(TApplicationException.MISSING_RESULT, \"" << (*f_iter)->get_name() << " failed: unknown result\");" << endl; + } + + // Close function + scope_down(f_service_); + f_service_ << endl; + } + } + + indent_down(); + indent(f_service_) << + "}" << endl; +} + +/** + * Generates a service server definition. + * + * @param tservice The service to generate a server for. + */ +void t_java_generator::generate_service_server(t_service* tservice) { + // Generate the dispatch methods + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + // Extends stuff + string extends = ""; + string extends_processor = ""; + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + extends_processor = " extends " + extends + ".Processor"; + } + + // Generate the header portion + indent(f_service_) << + "public static class Processor" << extends_processor << " implements TProcessor {" << endl; + indent_up(); + + indent(f_service_) << + "public Processor(Iface iface)" << endl; + scope_up(f_service_); + if (!extends.empty()) { + f_service_ << + indent() << "super(iface);" << endl; + } + f_service_ << + indent() << "iface_ = iface;" << endl; + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_service_ << + indent() << "processMap_.put(\"" << (*f_iter)->get_name() << "\", new " << (*f_iter)->get_name() << "());" << endl; + } + + scope_down(f_service_); + f_service_ << endl; + + if (extends.empty()) { + f_service_ << + indent() << "protected static interface ProcessFunction {" << endl << + indent() << " public void process(int seqid, TProtocol iprot, TProtocol oprot) throws TException;" << endl << + indent() << "}" << endl << + endl; + } + + f_service_ << + indent() << "private Iface iface_;" << endl; + + if (extends.empty()) { + f_service_ << + indent() << "protected final HashMap processMap_ = new HashMap();" << endl; + } + + f_service_ << endl; + + // Generate the server implementation + indent(f_service_) << + "public boolean process(TProtocol iprot, TProtocol oprot) throws TException" << endl; + scope_up(f_service_); + + f_service_ << + indent() << "TMessage msg = iprot.readMessageBegin();" << endl; + + // TODO(mcslee): validate message, was the seqid etc. legit? + + f_service_ << + indent() << "ProcessFunction fn = processMap_.get(msg.name);" << endl << + indent() << "if (fn == null) {" << endl << + indent() << " TProtocolUtil.skip(iprot, TType.STRUCT);" << endl << + indent() << " iprot.readMessageEnd();" << endl << + indent() << " TApplicationException x = new TApplicationException(TApplicationException.UNKNOWN_METHOD, \"Invalid method name: '\"+msg.name+\"'\");" << endl << + indent() << " oprot.writeMessageBegin(new TMessage(msg.name, TMessageType.EXCEPTION, msg.seqid));" << endl << + indent() << " x.write(oprot);" << endl << + indent() << " oprot.writeMessageEnd();" << endl << + indent() << " oprot.getTransport().flush();" << endl << + indent() << " return true;" << endl << + indent() << "}" << endl << + indent() << "fn.process(msg.seqid, iprot, oprot);" << endl; + + f_service_ << + indent() << "return true;" << endl; + + scope_down(f_service_); + f_service_ << endl; + + // Generate the process subfunctions + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + generate_process_function(tservice, *f_iter); + } + + indent_down(); + indent(f_service_) << + "}" << endl << + endl; +} + +/** + * Generates a struct and helpers for a function. + * + * @param tfunction The function + */ +void t_java_generator::generate_function_helpers(t_function* tfunction) { + if (tfunction->is_oneway()) { + return; + } + + t_struct result(program_, tfunction->get_name() + "_result"); + t_field success(tfunction->get_returntype(), "success", 0); + if (!tfunction->get_returntype()->is_void()) { + result.append(&success); + } + + t_struct* xs = tfunction->get_xceptions(); + const vector& fields = xs->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + result.append(*f_iter); + } + + generate_java_struct_definition(f_service_, &result, false, true, true); +} + +/** + * Generates a process function definition. + * + * @param tfunction The function to write a dispatcher for + */ +void t_java_generator::generate_process_function(t_service* tservice, + t_function* tfunction) { + // Open class + indent(f_service_) << + "private class " << tfunction->get_name() << " implements ProcessFunction {" << endl; + indent_up(); + + // Open function + indent(f_service_) << + "public void process(int seqid, TProtocol iprot, TProtocol oprot) throws TException" << endl; + scope_up(f_service_); + + string argsname = tfunction->get_name() + "_args"; + string resultname = tfunction->get_name() + "_result"; + + f_service_ << + indent() << argsname << " args = new " << argsname << "();" << endl << + indent() << "args.read(iprot);" << endl << + indent() << "iprot.readMessageEnd();" << endl; + + t_struct* xs = tfunction->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + + // Declare result for non oneway function + if (!tfunction->is_oneway()) { + f_service_ << + indent() << resultname << " result = new " << resultname << "();" << endl; + } + + // Try block for a function with exceptions + if (xceptions.size() > 0) { + f_service_ << + indent() << "try {" << endl; + indent_up(); + } + + // Generate the function call + t_struct* arg_struct = tfunction->get_arglist(); + const std::vector& fields = arg_struct->get_members(); + vector::const_iterator f_iter; + + f_service_ << indent(); + if (!tfunction->is_oneway() && !tfunction->get_returntype()->is_void()) { + f_service_ << "result.success = "; + } + f_service_ << + "iface_." << tfunction->get_name() << "("; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << "args." << (*f_iter)->get_name(); + } + f_service_ << ");" << endl; + + // Set isset on success field + if (!tfunction->is_oneway() && !tfunction->get_returntype()->is_void() && !type_can_be_null(tfunction->get_returntype())) { + f_service_ << + indent() << "result.__isset.success = true;" << endl; + } + + if (!tfunction->is_oneway() && xceptions.size() > 0) { + indent_down(); + f_service_ << indent() << "}"; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << " catch (" << type_name((*x_iter)->get_type(), false, false) << " " << (*x_iter)->get_name() << ") {" << endl; + if (!tfunction->is_oneway()) { + indent_up(); + f_service_ << + indent() << "result." << (*x_iter)->get_name() << " = " << (*x_iter)->get_name() << ";" << endl; + indent_down(); + f_service_ << indent() << "}"; + } else { + f_service_ << "}"; + } + } + f_service_ << endl; + } + + // Shortcut out here for oneway functions + if (tfunction->is_oneway()) { + f_service_ << + indent() << "return;" << endl; + scope_down(f_service_); + + // Close class + indent_down(); + f_service_ << + indent() << "}" << endl << + endl; + return; + } + + f_service_ << + indent() << "oprot.writeMessageBegin(new TMessage(\"" << tfunction->get_name() << "\", TMessageType.REPLY, seqid));" << endl << + indent() << "result.write(oprot);" << endl << + indent() << "oprot.writeMessageEnd();" << endl << + indent() << "oprot.getTransport().flush();" << endl; + + // Close function + scope_down(f_service_); + f_service_ << endl; + + // Close class + indent_down(); + f_service_ << + indent() << "}" << endl << + endl; +} + +/** + * Deserializes a field of any type. + * + * @param tfield The field + * @param prefix The variable name or container for this field + */ +void t_java_generator::generate_deserialize_field(ofstream& out, + t_field* tfield, + string prefix) { + t_type* type = get_true_type(tfield->get_type()); + + if (type->is_void()) { + throw "CANNOT GENERATE DESERIALIZE CODE FOR void TYPE: " + + prefix + tfield->get_name(); + } + + string name = prefix + tfield->get_name(); + + if (type->is_struct() || type->is_xception()) { + generate_deserialize_struct(out, + (t_struct*)type, + name); + } else if (type->is_container()) { + generate_deserialize_container(out, type, name); + } else if (type->is_base_type() || type->is_enum()) { + + indent(out) << + name << " = iprot."; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "compiler error: cannot serialize void field in a struct: " + + name; + break; + case t_base_type::TYPE_STRING: + if (((t_base_type*)type)->is_binary()) { + out << "readBinary();"; + } else { + out << "readString();"; + } + break; + case t_base_type::TYPE_BOOL: + out << "readBool();"; + break; + case t_base_type::TYPE_BYTE: + out << "readByte();"; + break; + case t_base_type::TYPE_I16: + out << "readI16();"; + break; + case t_base_type::TYPE_I32: + out << "readI32();"; + break; + case t_base_type::TYPE_I64: + out << "readI64();"; + break; + case t_base_type::TYPE_DOUBLE: + out << "readDouble();"; + break; + default: + throw "compiler error: no Java name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + out << "readI32();"; + } + out << + endl; + } else { + printf("DO NOT KNOW HOW TO DESERIALIZE FIELD '%s' TYPE '%s'\n", + tfield->get_name().c_str(), type_name(type).c_str()); + } +} + +/** + * Generates an unserializer for a struct, invokes read() + */ +void t_java_generator::generate_deserialize_struct(ofstream& out, + t_struct* tstruct, + string prefix) { + out << + indent() << prefix << " = new " << type_name(tstruct) << "();" << endl << + indent() << prefix << ".read(iprot);" << endl; +} + +/** + * Deserializes a container by reading its size and then iterating + */ +void t_java_generator::generate_deserialize_container(ofstream& out, + t_type* ttype, + string prefix) { + scope_up(out); + + string obj; + + if (ttype->is_map()) { + obj = tmp("_map"); + } else if (ttype->is_set()) { + obj = tmp("_set"); + } else if (ttype->is_list()) { + obj = tmp("_list"); + } + + // Declare variables, read header + if (ttype->is_map()) { + indent(out) << "TMap " << obj << " = iprot.readMapBegin();" << endl; + } else if (ttype->is_set()) { + indent(out) << "TSet " << obj << " = iprot.readSetBegin();" << endl; + } else if (ttype->is_list()) { + indent(out) << "TList " << obj << " = iprot.readListBegin();" << endl; + } + + indent(out) + << prefix << " = new " << type_name(ttype, false, true) + // size the collection correctly + << "(" + << (ttype->is_list() ? "" : "2*" ) + << obj << ".size" + << ");" << endl; + + // For loop iterates over elements + string i = tmp("_i"); + indent(out) << + "for (int " << i << " = 0; " << + i << " < " << obj << ".size" << "; " << + "++" << i << ")" << endl; + + scope_up(out); + + if (ttype->is_map()) { + generate_deserialize_map_element(out, (t_map*)ttype, prefix); + } else if (ttype->is_set()) { + generate_deserialize_set_element(out, (t_set*)ttype, prefix); + } else if (ttype->is_list()) { + generate_deserialize_list_element(out, (t_list*)ttype, prefix); + } + + scope_down(out); + + // Read container end + if (ttype->is_map()) { + indent(out) << "iprot.readMapEnd();" << endl; + } else if (ttype->is_set()) { + indent(out) << "iprot.readSetEnd();" << endl; + } else if (ttype->is_list()) { + indent(out) << "iprot.readListEnd();" << endl; + } + + scope_down(out); +} + + +/** + * Generates code to deserialize a map + */ +void t_java_generator::generate_deserialize_map_element(ofstream& out, + t_map* tmap, + string prefix) { + string key = tmp("_key"); + string val = tmp("_val"); + t_field fkey(tmap->get_key_type(), key); + t_field fval(tmap->get_val_type(), val); + + indent(out) << + declare_field(&fkey) << endl; + indent(out) << + declare_field(&fval) << endl; + + generate_deserialize_field(out, &fkey); + generate_deserialize_field(out, &fval); + + indent(out) << + prefix << ".put(" << key << ", " << val << ");" << endl; +} + +/** + * Deserializes a set element + */ +void t_java_generator::generate_deserialize_set_element(ofstream& out, + t_set* tset, + string prefix) { + string elem = tmp("_elem"); + t_field felem(tset->get_elem_type(), elem); + + indent(out) << + declare_field(&felem) << endl; + + generate_deserialize_field(out, &felem); + + indent(out) << + prefix << ".add(" << elem << ");" << endl; +} + +/** + * Deserializes a list element + */ +void t_java_generator::generate_deserialize_list_element(ofstream& out, + t_list* tlist, + string prefix) { + string elem = tmp("_elem"); + t_field felem(tlist->get_elem_type(), elem); + + indent(out) << + declare_field(&felem) << endl; + + generate_deserialize_field(out, &felem); + + indent(out) << + prefix << ".add(" << elem << ");" << endl; +} + + +/** + * Serializes a field of any type. + * + * @param tfield The field to serialize + * @param prefix Name to prepend to field name + */ +void t_java_generator::generate_serialize_field(ofstream& out, + t_field* tfield, + string prefix) { + t_type* type = get_true_type(tfield->get_type()); + + // Do nothing for void types + if (type->is_void()) { + throw "CANNOT GENERATE SERIALIZE CODE FOR void TYPE: " + + prefix + tfield->get_name(); + } + + if (type->is_struct() || type->is_xception()) { + generate_serialize_struct(out, + (t_struct*)type, + prefix + tfield->get_name()); + } else if (type->is_container()) { + generate_serialize_container(out, + type, + prefix + tfield->get_name()); + } else if (type->is_base_type() || type->is_enum()) { + + string name = prefix + tfield->get_name(); + indent(out) << + "oprot."; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw + "compiler error: cannot serialize void field in a struct: " + name; + break; + case t_base_type::TYPE_STRING: + if (((t_base_type*)type)->is_binary()) { + out << "writeBinary(" << name << ");"; + } else { + out << "writeString(" << name << ");"; + } + break; + case t_base_type::TYPE_BOOL: + out << "writeBool(" << name << ");"; + break; + case t_base_type::TYPE_BYTE: + out << "writeByte(" << name << ");"; + break; + case t_base_type::TYPE_I16: + out << "writeI16(" << name << ");"; + break; + case t_base_type::TYPE_I32: + out << "writeI32(" << name << ");"; + break; + case t_base_type::TYPE_I64: + out << "writeI64(" << name << ");"; + break; + case t_base_type::TYPE_DOUBLE: + out << "writeDouble(" << name << ");"; + break; + default: + throw "compiler error: no Java name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + out << "writeI32(" << name << ");"; + } + out << endl; + } else { + printf("DO NOT KNOW HOW TO SERIALIZE FIELD '%s%s' TYPE '%s'\n", + prefix.c_str(), + tfield->get_name().c_str(), + type_name(type).c_str()); + } +} + +/** + * Serializes all the members of a struct. + * + * @param tstruct The struct to serialize + * @param prefix String prefix to attach to all fields + */ +void t_java_generator::generate_serialize_struct(ofstream& out, + t_struct* tstruct, + string prefix) { + out << + indent() << prefix << ".write(oprot);" << endl; +} + +/** + * Serializes a container by writing its size then the elements. + * + * @param ttype The type of container + * @param prefix String prefix for fields + */ +void t_java_generator::generate_serialize_container(ofstream& out, + t_type* ttype, + string prefix) { + scope_up(out); + + if (ttype->is_map()) { + indent(out) << + "oprot.writeMapBegin(new TMap(" << + type_to_enum(((t_map*)ttype)->get_key_type()) << ", " << + type_to_enum(((t_map*)ttype)->get_val_type()) << ", " << + prefix << ".size()));" << endl; + } else if (ttype->is_set()) { + indent(out) << + "oprot.writeSetBegin(new TSet(" << + type_to_enum(((t_set*)ttype)->get_elem_type()) << ", " << + prefix << ".size()));" << endl; + } else if (ttype->is_list()) { + indent(out) << + "oprot.writeListBegin(new TList(" << + type_to_enum(((t_list*)ttype)->get_elem_type()) << ", " << + prefix << ".size()));" << endl; + } + + string iter = tmp("_iter"); + if (ttype->is_map()) { + indent(out) << + "for (Map.Entry<" << + type_name(((t_map*)ttype)->get_key_type(), true, false) << ", " << + type_name(((t_map*)ttype)->get_val_type(), true, false) << "> " << iter << + " : " << + prefix << ".entrySet())"; + } else if (ttype->is_set()) { + indent(out) << + "for (" << + type_name(((t_set*)ttype)->get_elem_type()) << " " << iter << + " : " << + prefix << ")"; + } else if (ttype->is_list()) { + indent(out) << + "for (" << + type_name(((t_list*)ttype)->get_elem_type()) << " " << iter << + " : " << + prefix << ")"; + } + + scope_up(out); + + if (ttype->is_map()) { + generate_serialize_map_element(out, (t_map*)ttype, iter, prefix); + } else if (ttype->is_set()) { + generate_serialize_set_element(out, (t_set*)ttype, iter); + } else if (ttype->is_list()) { + generate_serialize_list_element(out, (t_list*)ttype, iter); + } + + scope_down(out); + + if (ttype->is_map()) { + indent(out) << + "oprot.writeMapEnd();" << endl; + } else if (ttype->is_set()) { + indent(out) << + "oprot.writeSetEnd();" << endl; + } else if (ttype->is_list()) { + indent(out) << + "oprot.writeListEnd();" << endl; + } + + scope_down(out); +} + +/** + * Serializes the members of a map. + */ +void t_java_generator::generate_serialize_map_element(ofstream& out, + t_map* tmap, + string iter, + string map) { + t_field kfield(tmap->get_key_type(), iter + ".getKey()"); + generate_serialize_field(out, &kfield, ""); + t_field vfield(tmap->get_val_type(), iter + ".getValue()"); + generate_serialize_field(out, &vfield, ""); +} + +/** + * Serializes the members of a set. + */ +void t_java_generator::generate_serialize_set_element(ofstream& out, + t_set* tset, + string iter) { + t_field efield(tset->get_elem_type(), iter); + generate_serialize_field(out, &efield, ""); +} + +/** + * Serializes the members of a list. + */ +void t_java_generator::generate_serialize_list_element(ofstream& out, + t_list* tlist, + string iter) { + t_field efield(tlist->get_elem_type(), iter); + generate_serialize_field(out, &efield, ""); +} + +/** + * Returns a Java type name + * + * @param ttype The type + * @param container Is the type going inside a container? + * @return Java type name, i.e. HashMap + */ +string t_java_generator::type_name(t_type* ttype, bool in_container, bool in_init) { + // In Java typedefs are just resolved to their real type + ttype = get_true_type(ttype); + string prefix; + + if (ttype->is_base_type()) { + return base_type_name((t_base_type*)ttype, in_container); + } else if (ttype->is_enum()) { + return (in_container ? "Integer" : "int"); + } else if (ttype->is_map()) { + t_map* tmap = (t_map*) ttype; + if (in_init) { + prefix = "HashMap"; + } else { + prefix = "Map"; + } + return prefix + "<" + + type_name(tmap->get_key_type(), true) + "," + + type_name(tmap->get_val_type(), true) + ">"; + } else if (ttype->is_set()) { + t_set* tset = (t_set*) ttype; + if (in_init) { + prefix = "HashSet<"; + } else { + prefix = "Set<"; + } + return prefix + type_name(tset->get_elem_type(), true) + ">"; + } else if (ttype->is_list()) { + t_list* tlist = (t_list*) ttype; + if (in_init) { + prefix = "ArrayList<"; + } else { + prefix = "List<"; + } + return prefix + type_name(tlist->get_elem_type(), true) + ">"; + } + + // Check for namespacing + t_program* program = ttype->get_program(); + if (program != NULL && program != program_) { + string package = program->get_namespace("java"); + if (!package.empty()) { + return package + "." + ttype->get_name(); + } + } + + return ttype->get_name(); +} + +/** + * Returns the C++ type that corresponds to the thrift type. + * + * @param tbase The base type + * @param container Is it going in a Java container? + */ +string t_java_generator::base_type_name(t_base_type* type, + bool in_container) { + t_base_type::t_base tbase = type->get_base(); + + switch (tbase) { + case t_base_type::TYPE_VOID: + return "void"; + case t_base_type::TYPE_STRING: + if (type->is_binary()) { + return "byte[]"; + } else { + return "String"; + } + case t_base_type::TYPE_BOOL: + return (in_container ? "Boolean" : "boolean"); + case t_base_type::TYPE_BYTE: + return (in_container ? "Byte" : "byte"); + case t_base_type::TYPE_I16: + return (in_container ? "Short" : "short"); + case t_base_type::TYPE_I32: + return (in_container ? "Integer" : "int"); + case t_base_type::TYPE_I64: + return (in_container ? "Long" : "long"); + case t_base_type::TYPE_DOUBLE: + return (in_container ? "Double" : "double"); + default: + throw "compiler error: no C++ name for base type " + t_base_type::t_base_name(tbase); + } +} + +/** + * Declares a field, which may include initialization as necessary. + * + * @param ttype The type + */ +string t_java_generator::declare_field(t_field* tfield, bool init) { + // TODO(mcslee): do we ever need to initialize the field? + string result = type_name(tfield->get_type()) + " " + tfield->get_name(); + if (init) { + t_type* ttype = get_true_type(tfield->get_type()); + if (ttype->is_base_type() && tfield->get_value() != NULL) { + ofstream dummy; + result += " = " + render_const_value(dummy, tfield->get_name(), ttype, tfield->get_value()); + } else if (ttype->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)ttype)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + result += " = null"; + break; + case t_base_type::TYPE_BOOL: + result += " = false"; + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + result += " = 0"; + break; + case t_base_type::TYPE_DOUBLE: + result += " = (double)0"; + break; + } + + } else if (ttype->is_enum()) { + result += " = 0"; + } else if (ttype->is_container()) { + result += " = new " + type_name(ttype, false, true) + "()"; + } else { + result += " = new " + type_name(ttype, false, true) + "()";; + } + } + return result + ";"; +} + +/** + * Renders a function signature of the form 'type name(args)' + * + * @param tfunction Function definition + * @return String of rendered function definition + */ +string t_java_generator::function_signature(t_function* tfunction, + string prefix) { + t_type* ttype = tfunction->get_returntype(); + std::string result = + type_name(ttype) + " " + prefix + tfunction->get_name() + "(" + argument_list(tfunction->get_arglist()) + ") throws "; + t_struct* xs = tfunction->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + result += type_name((*x_iter)->get_type(), false, false) + ", "; + } + result += "TException"; + return result; +} + +/** + * Renders a comma separated field list, with type names + */ +string t_java_generator::argument_list(t_struct* tstruct) { + string result = ""; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + result += ", "; + } + result += type_name((*f_iter)->get_type()) + " " + (*f_iter)->get_name(); + } + return result; +} + +/** + * Converts the parse type to a C++ enum string for the given type. + */ +string t_java_generator::type_to_enum(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + return "TType.STRING"; + case t_base_type::TYPE_BOOL: + return "TType.BOOL"; + case t_base_type::TYPE_BYTE: + return "TType.BYTE"; + case t_base_type::TYPE_I16: + return "TType.I16"; + case t_base_type::TYPE_I32: + return "TType.I32"; + case t_base_type::TYPE_I64: + return "TType.I64"; + case t_base_type::TYPE_DOUBLE: + return "TType.DOUBLE"; + } + } else if (type->is_enum()) { + return "TType.I32"; + } else if (type->is_struct() || type->is_xception()) { + return "TType.STRUCT"; + } else if (type->is_map()) { + return "TType.MAP"; + } else if (type->is_set()) { + return "TType.SET"; + } else if (type->is_list()) { + return "TType.LIST"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + +/** + * Applies the correct style to a string based on the value of nocamel_style_ + */ +std::string t_java_generator::get_cap_name(std::string name){ + if (nocamel_style_) { + return "_" + name; + } else { + name[0] = toupper(name[0]); + return name; + } +} + +string t_java_generator::constant_name(string name) { + string constant_name; + + bool is_first = true; + bool was_previous_char_upper = false; + for (string::iterator iter = name.begin(); iter != name.end(); ++iter) { + string::value_type character = (*iter); + + bool is_upper = isupper(character); + + if (is_upper && !is_first && !was_previous_char_upper) { + constant_name += '_'; + } + constant_name += toupper(character); + + is_first = false; + was_previous_char_upper = is_upper; + } + + return constant_name; +} + +/** + * Emits a JavaDoc comment if the provided object has a doc in Thrift + */ +void t_java_generator::generate_java_doc(ofstream &out, + t_doc* tdoc) { + if (tdoc->has_doc()) { + generate_docstring_comment(out, + "/**\n", + " * ", tdoc->get_doc(), + " */\n"); + } +} + +/** + * Emits a JavaDoc comment if the provided function object has a doc in Thrift + */ +void t_java_generator::generate_java_doc(ofstream &out, + t_function* tfunction) { + if (tfunction->has_doc()) { + stringstream ss; + ss << tfunction->get_doc(); + const vector& fields = tfunction->get_arglist()->get_members(); + vector::const_iterator p_iter; + for (p_iter = fields.begin(); p_iter != fields.end(); ++p_iter) { + t_field* p = *p_iter; + ss << "\n@param " << p->get_name(); + if (p->has_doc()) { + ss << " " << p->get_doc(); + } + } + generate_docstring_comment(out, + "/**\n", + " * ", ss.str(), + " */\n"); + } +} + +void t_java_generator::generate_deep_copy_container(ofstream &out, std::string source_name_p1, std::string source_name_p2, + std::string result_name, t_type* type) { + + t_container* container = (t_container*)type; + std::string source_name; + if (source_name_p2 == "") + source_name = source_name_p1; + else + source_name = source_name_p1 + "." + source_name_p2; + + indent(out) << type_name(type, true, false) << " " << result_name << " = new " << type_name(container, false, true) << "();" << endl; + + std::string iterator_element_name = source_name_p1 + "_element"; + std::string result_element_name = result_name + "_copy"; + + if(container->is_map()) { + t_type* key_type = ((t_map*)container)->get_key_type(); + t_type* val_type = ((t_map*)container)->get_val_type(); + + indent(out) << + "for (Map.Entry<" << type_name(key_type, true, false) << ", " << type_name(val_type, true, false) << "> " << iterator_element_name << " : " << source_name << ".entrySet()) {" << endl; + indent_up(); + + out << endl; + + indent(out) << type_name(key_type, true, false) << " " << iterator_element_name << "_key = " << iterator_element_name << ".getKey();" << endl; + indent(out) << type_name(val_type, true, false) << " " << iterator_element_name << "_value = " << iterator_element_name << ".getValue();" << endl; + + out << endl; + + if (key_type->is_container()) { + generate_deep_copy_container(out, iterator_element_name + "_key", "", result_element_name + "_key", key_type); + } else { + indent(out) << type_name(key_type, true, false) << " " << result_element_name << "_key = "; + generate_deep_copy_non_container(out, iterator_element_name + "_key", result_element_name + "_key", key_type); + out << ";" << endl; + } + + out << endl; + + if (val_type->is_container()) { + generate_deep_copy_container(out, iterator_element_name + "_value", "", result_element_name + "_value", val_type); + } else { + indent(out) << type_name(val_type, true, false) << " " << result_element_name << "_value = "; + generate_deep_copy_non_container(out, iterator_element_name + "_value", result_element_name + "_value", val_type); + out << ";" << endl; + } + + out << endl; + + indent(out) << result_name << ".put(" << result_element_name << "_key, " << result_element_name << "_value);" << endl; + + indent_down(); + indent(out) << "}" << endl; + + } else { + t_type* elem_type; + + if (container->is_set()) { + elem_type = ((t_set*)container)->get_elem_type(); + } else { + elem_type = ((t_list*)container)->get_elem_type(); + } + + indent(out) + << "for (" << type_name(elem_type, true, false) << " " << iterator_element_name << " : " << source_name << ") {" << endl; + + indent_up(); + + if (elem_type->is_container()) { + // recursive deep copy + generate_deep_copy_container(out, iterator_element_name, "", result_element_name, elem_type); + indent(out) << result_name << ".add(" << result_element_name << ");" << endl; + } else { + // iterative copy + if(((t_base_type*)elem_type)->is_binary()){ + indent(out) << "byte[] temp_binary_element = "; + generate_deep_copy_non_container(out, iterator_element_name, "temp_binary_element", elem_type); + out << ";" << endl; + indent(out) << result_name << ".add(temp_binary_element);" << endl; + } + else{ + indent(out) << result_name << ".add("; + generate_deep_copy_non_container(out, iterator_element_name, result_name, elem_type); + out << ");" << endl; + } + } + + indent_down(); + + indent(out) << "}" << endl; + + } +} + +void t_java_generator::generate_deep_copy_non_container(ofstream& out, std::string source_name, std::string dest_name, t_type* type) { + if (type->is_base_type() || type->is_enum() || type->is_typedef()) { + // binary fields need to be copied with System.arraycopy + if (((t_base_type*)type)->is_binary()){ + out << "new byte[" << source_name << ".length];" << endl; + indent(out) << "System.arraycopy(" << source_name << ", 0, " << dest_name << ", 0, " << source_name << ".length)"; + } + // everything else can be copied directly + else + out << source_name; + } else { + out << "new " << type_name(type, true, true) << "(" << source_name << ")"; + } +} + +std::string t_java_generator::generate_isset_check(t_field* field) { + return generate_isset_check(field->get_name()); +} + +std::string t_java_generator::generate_isset_check(std::string field_name) { + return "is" + get_cap_name("set") + get_cap_name(field_name) + "()"; +} + +void t_java_generator::generate_isset_set(ofstream& out, t_field* field) { + if (!type_can_be_null(field->get_type())) { + indent(out) << "this.__isset." << field->get_name() << " = true;" << endl; + } +} + +std::string t_java_generator::get_enum_class_name(t_type* type) { + string package = ""; + t_program* program = type->get_program(); + if (program != NULL && program != program_) { + package = program->get_namespace("java") + "."; + } + return package + type->get_name(); +} + +THRIFT_REGISTER_GENERATOR(java, "Java", +" beans: Generate bean-style output files.\n" +" nocamel: Do not use CamelCase field accessors with beans.\n" +" hashcode: Generate quality hashCode methods.\n" +); diff --git a/compiler/cpp/src/generate/t_ocaml_generator.cc b/compiler/cpp/src/generate/t_ocaml_generator.cc new file mode 100644 index 00000000..0405a04f --- /dev/null +++ b/compiler/cpp/src/generate/t_ocaml_generator.cc @@ -0,0 +1,1673 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include "t_oop_generator.h" +#include "platform.h" +using namespace std; + + +/** + * OCaml code generator. + * + */ +class t_ocaml_generator : public t_oop_generator { + public: + t_ocaml_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_oop_generator(program) + { + out_dir_base_ = "gen-ocaml"; + } + + /** + * Init and close methods + */ + + void init_generator(); + void close_generator(); + + /** + * Program-level generation functions + */ + void generate_program (); + void generate_typedef (t_typedef* ttypedef); + void generate_enum (t_enum* tenum); + void generate_const (t_const* tconst); + void generate_struct (t_struct* tstruct); + void generate_xception (t_struct* txception); + void generate_service (t_service* tservice); + + std::string render_const_value(t_type* type, t_const_value* value); + + /** + * Struct generation code + */ + + void generate_ocaml_struct(t_struct* tstruct, bool is_exception); + void generate_ocaml_struct_definition(std::ofstream& out, t_struct* tstruct, bool is_xception=false); + void generate_ocaml_struct_sig(std::ofstream& out, t_struct* tstruct, bool is_exception); + void generate_ocaml_struct_reader(std::ofstream& out, t_struct* tstruct); + void generate_ocaml_struct_writer(std::ofstream& out, t_struct* tstruct); + void generate_ocaml_function_helpers(t_function* tfunction); + + /** + * Service-level generation functions + */ + + void generate_service_helpers (t_service* tservice); + void generate_service_interface (t_service* tservice); + void generate_service_client (t_service* tservice); + void generate_service_server (t_service* tservice); + void generate_process_function (t_service* tservice, t_function* tfunction); + + /** + * Serialization constructs + */ + + void generate_deserialize_field (std::ofstream &out, + t_field* tfield, + std::string prefix); + + void generate_deserialize_struct (std::ofstream &out, + t_struct* tstruct); + + void generate_deserialize_container (std::ofstream &out, + t_type* ttype); + + void generate_deserialize_set_element (std::ofstream &out, + t_set* tset); + + + void generate_deserialize_list_element (std::ofstream &out, + t_list* tlist, + std::string prefix=""); + void generate_deserialize_type (std::ofstream &out, + t_type* type); + + void generate_serialize_field (std::ofstream &out, + t_field* tfield, + std::string name= ""); + + void generate_serialize_struct (std::ofstream &out, + t_struct* tstruct, + std::string prefix=""); + + void generate_serialize_container (std::ofstream &out, + t_type* ttype, + std::string prefix=""); + + void generate_serialize_map_element (std::ofstream &out, + t_map* tmap, + std::string kiter, + std::string viter); + + void generate_serialize_set_element (std::ofstream &out, + t_set* tmap, + std::string iter); + + void generate_serialize_list_element (std::ofstream &out, + t_list* tlist, + std::string iter); + + /** + * Helper rendering functions + */ + + std::string ocaml_autogen_comment(); + std::string ocaml_imports(); + std::string type_name(t_type* ttype); + std::string function_signature(t_function* tfunction, std::string prefix=""); + std::string function_type(t_function* tfunc, bool method=false, bool options = false); + std::string argument_list(t_struct* tstruct); + std::string type_to_enum(t_type* ttype); + std::string render_ocaml_type(t_type* type); + + + private: + + /** + * File streams + */ + + std::ofstream f_types_; + std::ofstream f_consts_; + std::ofstream f_service_; + + std::ofstream f_types_i_; + std::ofstream f_service_i_; + +}; + + +/* + * This is necessary because we want typedefs to appear later, + * after all the types have been declared. + */ +void t_ocaml_generator::generate_program() { + // Initialize the generator + init_generator(); + + // Generate enums + vector enums = program_->get_enums(); + vector::iterator en_iter; + for (en_iter = enums.begin(); en_iter != enums.end(); ++en_iter) { + generate_enum(*en_iter); + } + + // Generate structs + vector structs = program_->get_structs(); + vector::iterator st_iter; + for (st_iter = structs.begin(); st_iter != structs.end(); ++st_iter) { + generate_struct(*st_iter); + } + + // Generate xceptions + vector xceptions = program_->get_xceptions(); + vector::iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + generate_xception(*x_iter); + } + + // Generate typedefs + vector typedefs = program_->get_typedefs(); + vector::iterator td_iter; + for (td_iter = typedefs.begin(); td_iter != typedefs.end(); ++td_iter) { + generate_typedef(*td_iter); + } + + // Generate services + vector services = program_->get_services(); + vector::iterator sv_iter; + for (sv_iter = services.begin(); sv_iter != services.end(); ++sv_iter) { + service_name_ = get_service_name(*sv_iter); + generate_service(*sv_iter); + } + + // Generate constants + vector consts = program_->get_consts(); + generate_consts(consts); + + // Close the generator + close_generator(); +} + + +/** + * Prepares for file generation by opening up the necessary file output + * streams. + * + * @param tprogram The program to generate + */ +void t_ocaml_generator::init_generator() { + // Make output directory + MKDIR(get_out_dir().c_str()); + + // Make output file + string f_types_name = get_out_dir()+program_name_+"_types.ml"; + f_types_.open(f_types_name.c_str()); + string f_types_i_name = get_out_dir()+program_name_+"_types.mli"; + f_types_i_.open(f_types_i_name.c_str()); + + string f_consts_name = get_out_dir()+program_name_+"_consts.ml"; + f_consts_.open(f_consts_name.c_str()); + + // Print header + f_types_ << + ocaml_autogen_comment() << endl << + ocaml_imports() << endl; + f_types_i_ << + ocaml_autogen_comment() << endl << + ocaml_imports() << endl; + f_consts_ << + ocaml_autogen_comment() << endl << + ocaml_imports() << endl << + "open " << capitalize(program_name_)<<"_types"<< endl; +} + + +/** + * Autogen'd comment + */ +string t_ocaml_generator::ocaml_autogen_comment() { + return + std::string("(*\n") + + " Autogenerated by Thrift\n" + + "\n" + + " DO NOT EDIT UNLESS YOU ARE SURE YOU KNOW WHAT YOU ARE DOING\n" + + "*)\n"; +} + +/** + * Prints standard thrift imports + */ +string t_ocaml_generator::ocaml_imports() { + return "open Thrift"; +} + +/** + * Closes the type files + */ +void t_ocaml_generator::close_generator() { + // Close types file + f_types_.close(); +} + +/** + * Generates a typedef. Ez. + * + * @param ttypedef The type definition + */ +void t_ocaml_generator::generate_typedef(t_typedef* ttypedef) { + f_types_ << + indent() << "type "<< decapitalize(ttypedef->get_symbolic()) << " = " << render_ocaml_type(ttypedef->get_type()) << endl << endl; + f_types_i_ << + indent() << "type "<< decapitalize(ttypedef->get_symbolic()) << " = " << render_ocaml_type(ttypedef->get_type()) << endl << endl; +} + +/** + * Generates code for an enumerated type. + * the values. + * + * @param tenum The enumeration + */ +void t_ocaml_generator::generate_enum(t_enum* tenum) { + indent(f_types_) << "module " << capitalize(tenum->get_name()) << " = " << endl << "struct" << endl; + indent(f_types_i_) << "module " << capitalize(tenum->get_name()) << " : " << endl << "sig" << endl; + indent_up(); + indent(f_types_) << "type t = " << endl; + indent(f_types_i_) << "type t = " << endl; + indent_up(); + vector constants = tenum->get_constants(); + vector::iterator c_iter; + int value = -1; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + string name = capitalize((*c_iter)->get_name()); + indent(f_types_) << "| " << name << endl; + indent(f_types_i_) << "| " << name << endl; + } + indent_down(); + + indent(f_types_) << "let to_i = function" << endl; + indent(f_types_i_) << "val to_i : t -> int" << endl; + indent_up(); + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + string name = capitalize((*c_iter)->get_name()); + + f_types_ << + indent() << "| " << name << " -> " << value << endl; + } + indent_down(); + + indent(f_types_) << "let of_i = function" << endl; + indent(f_types_i_) << "val of_i : int -> t" << endl; + indent_up(); + for(c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + string name = capitalize((*c_iter)->get_name()); + + f_types_ << + indent() << "| " << value << " -> " << name << endl; + } + indent(f_types_) << "| _ -> raise Thrift_error" << endl; + indent_down(); + indent_down(); + indent(f_types_) << "end" << endl; + indent(f_types_i_) << "end" << endl; +} + +/** + * Generate a constant value + */ +void t_ocaml_generator::generate_const(t_const* tconst) { + t_type* type = tconst->get_type(); + string name = decapitalize(tconst->get_name()); + t_const_value* value = tconst->get_value(); + + indent(f_consts_) << "let " << name << " = " << render_const_value(type, value) << endl << endl; +} + +/** + * Prints the value of a constant with the given type. Note that type checking + * is NOT performed in this function as it is always run beforehand using the + * validate_types method in main.cc + */ +string t_ocaml_generator::render_const_value(t_type* type, t_const_value* value) { + type = get_true_type(type); + std::ostringstream out; + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + out << '"' << get_escaped_string(value) << '"'; + break; + case t_base_type::TYPE_BOOL: + out << (value->get_integer() > 0 ? "true" : "false"); + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + out << value->get_integer(); + break; + case t_base_type::TYPE_I64: + out << value->get_integer() << "L"; + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() == t_const_value::CV_INTEGER) { + out << value->get_integer(); + } else { + out << value->get_double(); + } + break; + default: + throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + t_enum* tenum = (t_enum*)type; + vector constants = tenum->get_constants(); + vector::iterator c_iter; + int val = -1; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + val = (*c_iter)->get_value(); + } else { + ++val; + } + if(val == value->get_integer()){ + indent(out) << capitalize(tenum->get_name()) << "." << capitalize((*c_iter)->get_name()); + break; + } + } + } else if (type->is_struct() || type->is_xception()) { + string cname = type_name(type); + string ct = tmp("_c"); + out << endl; + indent_up(); + indent(out) << "(let " << ct << " = new " << cname << " in" << endl; + indent_up(); + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + string fname = v_iter->first->get_string(); + out << indent(); + out << ct <<"#set_" << fname << " "; + out << render_const_value(field_type, v_iter->second); + out << ";" << endl; + } + indent(out) << ct << ")"; + indent_down(); + indent_down(); + } else if (type->is_map()) { + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + const map& val = value->get_map(); + map::const_iterator v_iter; + string hm = tmp("_hm"); + out << endl; + indent_up(); + indent(out) << "(let " << hm << " = Hashtbl.create " << val.size() << " in" << endl; + indent_up(); + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + string key = render_const_value(ktype, v_iter->first); + string val = render_const_value(vtype, v_iter->second); + indent(out) << "Hashtbl.add " << hm << " " << key << " " << val << ";" << endl; + } + indent(out) << hm << ")"; + indent_down(); + indent_down(); + } else if (type->is_list()) { + t_type* etype; + etype = ((t_list*)type)->get_elem_type(); + out << "[" << endl; + indent_up(); + const vector& val = value->get_list(); + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + out << indent(); + out << render_const_value(etype, *v_iter); + out << ";" << endl; + } + indent_down(); + indent(out) << "]"; + } else if (type->is_set()) { + t_type* etype = ((t_set*)type)->get_elem_type(); + const vector& val = value->get_list(); + vector::const_iterator v_iter; + string hm = tmp("_hm"); + indent(out) << "(let " << hm << " = Hashtbl.create " << val.size() << " in" << endl; + indent_up(); + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + string val = render_const_value(etype, *v_iter); + indent(out) << "Hashtbl.add " << hm << " " << val << " true;" << endl; + } + indent(out) << hm << ")" << endl; + indent_down(); + out << endl; + } else { + throw "CANNOT GENERATE CONSTANT FOR TYPE: " + type->get_name(); + } + return out.str(); +} + +/** + * Generates a "struct" + */ +void t_ocaml_generator::generate_struct(t_struct* tstruct) { + generate_ocaml_struct(tstruct, false); +} + +/** + * Generates a struct definition for a thrift exception. Basically the same + * as a struct, but also has an exception declaration. + * + * @param txception The struct definition + */ +void t_ocaml_generator::generate_xception(t_struct* txception) { + generate_ocaml_struct(txception, true); +} + +/** + * Generates an OCaml struct + */ +void t_ocaml_generator::generate_ocaml_struct(t_struct* tstruct, + bool is_exception) { + generate_ocaml_struct_definition(f_types_, tstruct, is_exception); + generate_ocaml_struct_sig(f_types_i_,tstruct,is_exception); +} + +/** + * Generates a struct definition for a thrift data type. + * + * @param tstruct The struct definition + */ +void t_ocaml_generator::generate_ocaml_struct_definition(ofstream& out, + t_struct* tstruct, + bool is_exception) { + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + string tname = type_name(tstruct); + indent(out) << "class " << tname << " =" << endl; + indent(out) << "object (self)" << endl; + + indent_up(); + + string x = tmp("_x"); + if (members.size() > 0) { + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + string mname = decapitalize((*m_iter)->get_name()); + indent(out) << "val mutable _" << mname << " : " << render_ocaml_type((*m_iter)->get_type()) << " option = None" << endl; + indent(out) << "method get_" << mname << " = _" << mname << endl; + indent(out) << "method grab_" << mname << " = match _"<raise (Field_empty \""< " << x << endl; + indent(out) << "method set_" << mname << " " << x << " = _" << mname << " <- Some " << x << endl; + } + } + generate_ocaml_struct_writer(out, tstruct); + indent_down(); + indent(out) << "end" << endl; + + if(is_exception){ + indent(out) << "exception " << capitalize(tname) <<" of " << tname << endl; + } + + generate_ocaml_struct_reader(out, tstruct); +} + +/** + * Generates a struct definition for a thrift data type. + * + * @param tstruct The struct definition + */ +void t_ocaml_generator::generate_ocaml_struct_sig(ofstream& out, + t_struct* tstruct, + bool is_exception) { + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + string tname = type_name(tstruct); + indent(out) << "class " << tname << " :" << endl; + indent(out) << "object" << endl; + + indent_up(); + + string x = tmp("_x"); + if (members.size() > 0) { + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + string mname = decapitalize((*m_iter)->get_name()); + string type = render_ocaml_type((*m_iter)->get_type()); + indent(out) << "method get_" << mname << " : " << type << " option" << endl; + indent(out) << "method grab_" << mname << " : " << type << endl; + indent(out) << "method set_" << mname << " : " << type << " -> unit" << endl; + } + } + indent(out) << "method write : Protocol.t -> unit" << endl; + indent_down(); + indent(out) << "end" << endl; + + if(is_exception){ + indent(out) << "exception " << capitalize(tname) <<" of " << tname << endl; + } + + indent(out) << "val read_" << tname << " : Protocol.t -> " << tname << endl; +} + +/** + * Generates the read method for a struct + */ +void t_ocaml_generator::generate_ocaml_struct_reader(ofstream& out, t_struct* tstruct) { + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + string sname = type_name(tstruct); + string str = tmp("_str"); + string t = tmp("_t"); + string id = tmp("_id"); + indent(out) << + "let rec read_" << sname << " (iprot : Protocol.t) =" << endl; + indent_up(); + indent(out) << "let " << str << " = new " << sname << " in" << endl; + indent_up(); + indent(out) << + "ignore(iprot#readStructBegin);" << endl; + + // Loop over reading in fields + indent(out) << + "(try while true do" << endl; + indent_up(); + indent_up(); + + // Read beginning field marker + indent(out) << + "let (_," << t <<","<get_key() << " -> ("; + out << "if " << t <<" = " << type_to_enum((*f_iter)->get_type()) << " then" << endl; + indent_up(); + indent_up(); + generate_deserialize_field(out, *f_iter,str); + indent_down(); + out << + indent() << "else" << endl << + indent() << " iprot#skip "<< t << ")" << endl; + indent_down(); + } + + // In the default case we skip the field + out << + indent() << "| _ -> " << "iprot#skip "< ());" << endl; + + indent(out) << + "iprot#readStructEnd;" << endl; + + indent(out) << str << endl << endl; + indent_down(); + indent_down(); +} + +void t_ocaml_generator::generate_ocaml_struct_writer(ofstream& out, + t_struct* tstruct) { + string name = tstruct->get_name(); + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + string str = tmp("_str"); + string f = tmp("_f"); + + indent(out) << + "method write (oprot : Protocol.t) =" << endl; + indent_up(); + indent(out) << + "oprot#writeStructBegin \""<get_name()); + indent(out) << + "(match " << mname << " with None -> () | Some _v -> " << endl; + indent_up(); + indent(out) << "oprot#writeFieldBegin(\""<< (*f_iter)->get_name()<<"\"," + <get_type())<<"," + <<(*f_iter)->get_key()<<");" << endl; + + // Write field contents + generate_serialize_field(out, *f_iter, "_v"); + + // Write field closer + indent(out) << "oprot#writeFieldEnd" << endl; + + indent_down(); + indent(out) << ");" << endl; + } + + // Write the struct map + out << + indent() << "oprot#writeFieldStop;" << endl << + indent() << "oprot#writeStructEnd" << endl; + + indent_down(); +} + +/** + * Generates a thrift service. + * + * @param tservice The service definition + */ +void t_ocaml_generator::generate_service(t_service* tservice) { + string f_service_name = get_out_dir()+capitalize(service_name_)+".ml"; + f_service_.open(f_service_name.c_str()); + string f_service_i_name = get_out_dir()+capitalize(service_name_)+".mli"; + f_service_i_.open(f_service_i_name.c_str()); + + f_service_ << + ocaml_autogen_comment() << endl << + ocaml_imports() << endl; + f_service_i_ << + ocaml_autogen_comment() << endl << + ocaml_imports() << endl; + + /* if (tservice->get_extends() != NULL) { + f_service_ << + "open " << capitalize(tservice->get_extends()->get_name()) << endl; + f_service_i_ << + "open " << capitalize(tservice->get_extends()->get_name()) << endl; + } + */ + f_service_ << + "open " << capitalize(program_name_) << "_types" << endl << + endl; + + f_service_i_ << + "open " << capitalize(program_name_) << "_types" << endl << + endl; + + // Generate the three main parts of the service + generate_service_helpers(tservice); + generate_service_interface(tservice); + generate_service_client(tservice); + generate_service_server(tservice); + + + // Close service file + f_service_.close(); + f_service_i_.close(); +} + +/** + * Generates helper functions for a service. + * + * @param tservice The service to generate a header definition for + */ +void t_ocaml_generator::generate_service_helpers(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + indent(f_service_) << + "(* HELPER FUNCTIONS AND STRUCTURES *)" << endl << endl; + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* ts = (*f_iter)->get_arglist(); + generate_ocaml_struct_definition(f_service_, ts, false); + generate_ocaml_function_helpers(*f_iter); + } +} + +/** + * Generates a struct and helpers for a function. + * + * @param tfunction The function + */ +void t_ocaml_generator::generate_ocaml_function_helpers(t_function* tfunction) { + t_struct result(program_, decapitalize(tfunction->get_name()) + "_result"); + t_field success(tfunction->get_returntype(), "success", 0); + if (!tfunction->get_returntype()->is_void()) { + result.append(&success); + } + + t_struct* xs = tfunction->get_xceptions(); + const vector& fields = xs->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + result.append(*f_iter); + } + generate_ocaml_struct_definition(f_service_, &result, false); +} + +/** + * Generates a service interface definition. + * + * @param tservice The service to generate a header definition for + */ +void t_ocaml_generator::generate_service_interface(t_service* tservice) { + f_service_ << + indent() << "class virtual iface =" << endl << "object (self)" << endl; + f_service_i_ << + indent() << "class virtual iface :" << endl << "object" << endl; + + indent_up(); + + if (tservice->get_extends() != NULL) { + string extends = type_name(tservice->get_extends()); + indent(f_service_) << "inherit " << extends << ".iface" << endl; + indent(f_service_i_) << "inherit " << extends << ".iface" << endl; + } + + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string ft = function_type(*f_iter,true,true); + f_service_ << + indent() << "method virtual " << decapitalize((*f_iter)->get_name()) << " : " << ft << endl; + f_service_i_ << + indent() << "method virtual " << decapitalize((*f_iter)->get_name()) << " : " << ft << endl; + } + indent_down(); + indent(f_service_) << "end" << endl << endl; + indent(f_service_i_) << "end" << endl << endl; +} + +/** + * Generates a service client definition. Note that in OCaml, the client doesn't implement iface. This is because + * The client does not (and should not have to) deal with arguments being None. + * + * @param tservice The service to generate a server for. + */ +void t_ocaml_generator::generate_service_client(t_service* tservice) { + string extends = ""; + indent(f_service_) << + "class client (iprot : Protocol.t) (oprot : Protocol.t) =" << endl << "object (self)" << endl; + indent(f_service_i_) << + "class client : Protocol.t -> Protocol.t -> " << endl << "object" << endl; + indent_up(); + + + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + indent(f_service_) << "inherit " << extends << ".client iprot oprot as super" << endl; + indent(f_service_i_) << "inherit " << extends << ".client" << endl; + } + indent(f_service_) << "val mutable seqid = 0" << endl; + + + // Generate client method implementations + vector functions = tservice->get_functions(); + vector::const_iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* arg_struct = (*f_iter)->get_arglist(); + const vector& fields = arg_struct->get_members(); + vector::const_iterator fld_iter; + string funname = (*f_iter)->get_name(); + + // Open function + indent(f_service_) << + "method " << function_signature(*f_iter) << " = " << endl; + indent(f_service_i_) << + "method " << decapitalize((*f_iter)->get_name()) << " : " << function_type(*f_iter,true,false) << endl; + indent_up(); + indent(f_service_) << + "self#send_" << funname; + + + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + f_service_ << " " << decapitalize((*fld_iter)->get_name()); + } + f_service_ << ";" << endl; + + if (!(*f_iter)->is_oneway()) { + f_service_ << indent(); + f_service_ << + "self#recv_" << funname << endl; + } + indent_down(); + + indent(f_service_) << + "method private send_" << function_signature(*f_iter) << " = " << endl; + indent_up(); + + std::string argsname = decapitalize((*f_iter)->get_name() + "_args"); + + // Serialize the request header + f_service_ << + indent() << "oprot#writeMessageBegin (\"" << (*f_iter)->get_name() << "\", Protocol.CALL, seqid);" << endl; + + f_service_ << + indent() << "let args = new " << argsname << " in" << endl; + indent_up(); + + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + f_service_ << + indent() << "args#set_" << (*fld_iter)->get_name() << " " << (*fld_iter)->get_name() << ";" << endl; + } + + // Write to the stream + f_service_ << + indent() << "args#write oprot;" << endl << + indent() << "oprot#writeMessageEnd;" << endl << + indent() << "oprot#getTransport#flush" << endl; + + indent_down(); + indent_down(); + + if (!(*f_iter)->is_oneway()) { + std::string resultname = decapitalize((*f_iter)->get_name() + "_result"); + t_struct noargs(program_); + + t_function recv_function((*f_iter)->get_returntype(), + string("recv_") + (*f_iter)->get_name(), + &noargs); + // Open function + f_service_ << + indent() << "method private " << function_signature(&recv_function) << " =" << endl; + indent_up(); + + // TODO(mcslee): Validate message reply here, seq ids etc. + + f_service_ << + indent() << "let (fname, mtype, rseqid) = iprot#readMessageBegin in" << endl; + indent_up(); + f_service_ << + indent() << "(if mtype = Protocol.EXCEPTION then" << endl << + indent() << " let x = Application_Exn.read iprot in" << endl; + indent_up(); + f_service_ << + indent() << " (iprot#readMessageEnd;" << + indent() << " raise (Application_Exn.E x))" << endl; + indent_down(); + f_service_ << + indent() << "else ());" << endl; + string res = "_"; + + t_struct* xs = (*f_iter)->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + + if (!(*f_iter)->get_returntype()->is_void() || xceptions.size() > 0) { + res = "result"; + } + f_service_ << + indent() << "let "<get_returntype()->is_void()) { + f_service_ << + indent() << "match result#get_success with Some v -> v | None -> (" << endl; + indent_up(); + } + + + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << + indent() << "(match result#get_" << (*x_iter)->get_name() << " with None -> () | Some _v ->" << endl; + indent(f_service_) << " raise (" << capitalize(type_name((*x_iter)->get_type())) << " _v));" << endl; + } + + // Careful, only return _result if not a void function + if ((*f_iter)->get_returntype()->is_void()) { + indent(f_service_) << + "()" << endl; + } else { + f_service_ << + indent() << "raise (Application_Exn.E (Application_Exn.create Application_Exn.MISSING_RESULT \"" << (*f_iter)->get_name() << " failed: unknown result\")))" << endl; + indent_down(); + } + + // Close function + indent_down(); + indent_down(); + indent_down(); + } + } + + indent_down(); + indent(f_service_) << "end" << endl << endl; + indent(f_service_i_) << "end" << endl << endl; +} + +/** + * Generates a service server definition. + * + * @param tservice The service to generate a server for. + */ +void t_ocaml_generator::generate_service_server(t_service* tservice) { + // Generate the dispatch methods + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + + // Generate the header portion + indent(f_service_) << + "class processor (handler : iface) =" << endl << indent() << "object (self)" << endl; + indent(f_service_i_) << + "class processor : iface ->" << endl << indent() << "object" << endl; + indent_up(); + + f_service_ << + indent() << "inherit Processor.t" << endl << + endl; + f_service_i_ << + indent() << "inherit Processor.t" << endl << + endl; + string extends = ""; + + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + indent(f_service_) << "inherit " + extends + ".processor (handler :> " + extends + ".iface)" << endl; + indent(f_service_i_) << "inherit " + extends + ".processor" << endl; + } + + if (extends.empty()) { + indent(f_service_) << "val processMap = Hashtbl.create " << functions.size() << endl; + } + indent(f_service_i_) << "val processMap : (string, int * Protocol.t * Protocol.t -> unit) Hashtbl.t" << endl; + + // Generate the server implementation + indent(f_service_) << + "method process iprot oprot =" << endl; + indent(f_service_i_) << + "method process : Protocol.t -> Protocol.t -> bool" << endl; + indent_up(); + + f_service_ << + indent() << "let (name, typ, seqid) = iprot#readMessageBegin in" << endl; + indent_up(); + // TODO(mcslee): validate message + + // HOT: dictionary function lookup + f_service_ << + indent() << "if Hashtbl.mem processMap name then" << endl << + indent() << " (Hashtbl.find processMap name) (seqid, iprot, oprot)" << endl << + indent() << "else (" << endl << + indent() << " iprot#skip(Protocol.T_STRUCT);" << endl << + indent() << " iprot#readMessageEnd;" << endl << + indent() << " let x = Application_Exn.create Application_Exn.UNKNOWN_METHOD (\"Unknown function \"^name) in" << endl << + indent() << " oprot#writeMessageBegin(name, Protocol.EXCEPTION, seqid);" << endl << + indent() << " x#write oprot;" << endl << + indent() << " oprot#writeMessageEnd;" << endl << + indent() << " oprot#getTransport#flush" << endl << + indent() << ");" << endl; + + // Read end of args field, the T_STOP, and the struct close + f_service_ << + indent() << "true" << endl; + indent_down(); + indent_down(); + // Generate the process subfunctions + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + generate_process_function(tservice, *f_iter); + } + + indent(f_service_) << "initializer" << endl; + indent_up(); + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_service_ << + indent() << "Hashtbl.add processMap \"" << (*f_iter)->get_name() << "\" self#process_" << (*f_iter)->get_name() << ";" << endl; + } + indent_down(); + + indent_down(); + indent(f_service_) << "end" << endl << endl; + indent(f_service_i_) << "end" << endl << endl; +} + +/** + * Generates a process function definition. + * + * @param tfunction The function to write a dispatcher for + */ +void t_ocaml_generator::generate_process_function(t_service* tservice, + t_function* tfunction) { + // Open function + indent(f_service_) << + "method private process_" << tfunction->get_name() << + " (seqid, iprot, oprot) =" << endl; + indent_up(); + + string argsname = decapitalize(tfunction->get_name()) + "_args"; + string resultname = decapitalize(tfunction->get_name()) + "_result"; + + // Generate the function call + t_struct* arg_struct = tfunction->get_arglist(); + const std::vector& fields = arg_struct->get_members(); + vector::const_iterator f_iter; + + string args = "args"; + if(fields.size() == 0){ + args="_"; + } + + f_service_ << + indent() << "let "<get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + + // Declare result for non oneway function + if (!tfunction->is_oneway()) { + f_service_ << + indent() << "let result = new " << resultname << " in" << endl; + indent_up(); + } + + // Try block for a function with exceptions + if (xceptions.size() > 0) { + f_service_ << + indent() << "(try" << endl; + indent_up(); + } + + + + + f_service_ << indent(); + if (!tfunction->is_oneway() && !tfunction->get_returntype()->is_void()) { + f_service_ << "result#set_success "; + } + f_service_ << + "(handler#" << tfunction->get_name(); + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + f_service_ << " args#get_" << (*f_iter)->get_name(); + } + f_service_ << ");" << endl; + + + if (xceptions.size() > 0) { + indent_down(); + indent(f_service_) << "with" <get_type())) << " " << (*x_iter)->get_name() << " -> " << endl; + indent_up(); + indent_up(); + if(!tfunction->is_oneway()){ + f_service_ << + indent() << "result#set_" << (*x_iter)->get_name() << " " << (*x_iter)->get_name() << endl; + } else { + indent(f_service_) << "()"; + } + indent_down(); + indent_down(); + } + indent_down(); + f_service_ << indent() << ");" << endl; + } + + + + // Shortcut out here for oneway functions + if (tfunction->is_oneway()) { + f_service_ << + indent() << "()" << endl; + indent_down(); + indent_down(); + return; + } + + f_service_ << + indent() << "oprot#writeMessageBegin (\"" << tfunction->get_name() << "\", Protocol.REPLY, seqid);" << endl << + indent() << "result#write oprot;" << endl << + indent() << "oprot#writeMessageEnd;" << endl << + indent() << "oprot#getTransport#flush" << endl; + + // Close function + indent_down(); + indent_down(); + indent_down(); +} + +/** + * Deserializes a field of any type. + */ +void t_ocaml_generator::generate_deserialize_field(ofstream &out, + t_field* tfield, + string prefix){ + t_type* type = tfield->get_type(); + + + string name = decapitalize(tfield->get_name()); + indent(out) << prefix << "#set_"<is_void()) { + throw "CANNOT GENERATE DESERIALIZE CODE FOR void TYPE"; + } + + + if (type->is_struct() || type->is_xception()) { + generate_deserialize_struct(out, + (t_struct*)type); + } else if (type->is_container()) { + generate_deserialize_container(out, type); + } else if (type->is_base_type()) { + out << "iprot#"; + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "compiler error: cannot serialize void field in a struct"; + break; + case t_base_type::TYPE_STRING: + out << "readString"; + break; + case t_base_type::TYPE_BOOL: + out << "readBool"; + break; + case t_base_type::TYPE_BYTE: + out << "readByte"; + break; + case t_base_type::TYPE_I16: + out << "readI16"; + break; + case t_base_type::TYPE_I32: + out << "readI32"; + break; + case t_base_type::TYPE_I64: + out << "readI64"; + break; + case t_base_type::TYPE_DOUBLE: + out << "readDouble"; + break; + default: + throw "compiler error: no PHP name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + string ename = capitalize(type->get_name()); + out << "(" <get_name().c_str()); + } +} + + +/** + * Generates an unserializer for a struct, calling read() + */ +void t_ocaml_generator::generate_deserialize_struct(ofstream &out, + t_struct* tstruct) { + string name = decapitalize(tstruct->get_name()); + out << "(read_" << name << " iprot)"; + +} + +/** + * Serialize a container by writing out the header followed by + * data and then a footer. + */ +void t_ocaml_generator::generate_deserialize_container(ofstream &out, + t_type* ttype) { + string size = tmp("_size"); + string ktype = tmp("_ktype"); + string vtype = tmp("_vtype"); + string etype = tmp("_etype"); + string con = tmp("_con"); + + t_field fsize(g_type_i32, size); + t_field fktype(g_type_byte, ktype); + t_field fvtype(g_type_byte, vtype); + t_field fetype(g_type_byte, etype); + + out << endl; + indent_up(); + // Declare variables, read header + if (ttype->is_map()) { + indent(out) << "(let ("<is_set()) { + indent(out) << "(let ("<get_elem_type()); + out << " true" << endl; + indent_down(); + indent(out) << "done; iprot#readSetEnd; "<is_list()) { + indent(out) << "(let ("< "; + generate_deserialize_type(out,((t_list*)ttype)->get_elem_type()); + out << "))) in" << endl; + indent_up(); + indent(out) << "iprot#readListEnd; "<get_type()); + + // Do nothing for void types + if (type->is_void()) { + throw "CANNOT GENERATE SERIALIZE CODE FOR void TYPE: " + + tfield->get_name(); + } + + if(name.length() == 0){ + name = decapitalize(tfield->get_name()); + } + + if (type->is_struct() || type->is_xception()) { + generate_serialize_struct(out, + (t_struct*)type, + name); + } else if (type->is_container()) { + generate_serialize_container(out, + type, + name); + } else if (type->is_base_type() || type->is_enum()) { + + + indent(out) << + "oprot#"; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw + "compiler error: cannot serialize void field in a struct: " + name; + break; + case t_base_type::TYPE_STRING: + out << "writeString(" << name << ")"; + break; + case t_base_type::TYPE_BOOL: + out << "writeBool(" << name << ")"; + break; + case t_base_type::TYPE_BYTE: + out << "writeByte(" << name << ")"; + break; + case t_base_type::TYPE_I16: + out << "writeI16(" << name << ")"; + break; + case t_base_type::TYPE_I32: + out << "writeI32(" << name << ")"; + break; + case t_base_type::TYPE_I64: + out << "writeI64(" << name << ")"; + break; + case t_base_type::TYPE_DOUBLE: + out << "writeDouble(" << name << ")"; + break; + default: + throw "compiler error: no ocaml name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + string ename = capitalize(type->get_name()); + out << "writeI32("<get_name().c_str(), + type->get_name().c_str()); + } + out << ";" << endl; +} + +/** + * Serializes all the members of a struct. + * + * @param tstruct The struct to serialize + * @param prefix String prefix to attach to all fields + */ +void t_ocaml_generator::generate_serialize_struct(ofstream &out, + t_struct* tstruct, + string prefix) { + indent(out) << prefix << "#write(oprot)"; +} + +void t_ocaml_generator::generate_serialize_container(ofstream &out, + t_type* ttype, + string prefix) { + if (ttype->is_map()) { + indent(out) << "oprot#writeMapBegin("<< type_to_enum(((t_map*)ttype)->get_key_type()) << ","; + out << type_to_enum(((t_map*)ttype)->get_val_type()) << ","; + out << "Hashtbl.length " << prefix << ");" << endl; + } else if (ttype->is_set()) { + indent(out) << + "oprot#writeSetBegin(" << type_to_enum(((t_set*)ttype)->get_elem_type()) << ","; + out << "Hashtbl.length " << prefix << ");" << endl; + } else if (ttype->is_list()) { + indent(out) << + "oprot#writeListBegin(" << type_to_enum(((t_list*)ttype)->get_elem_type()) << ","; + out << "List.length " << prefix << ");" << endl; + } + + if (ttype->is_map()) { + string kiter = tmp("_kiter"); + string viter = tmp("_viter"); + indent(out) << "Hashtbl.iter (fun "< fun " << viter << " -> " << endl; + indent_up(); + generate_serialize_map_element(out, (t_map*)ttype, kiter, viter); + indent_down(); + indent(out) << ") " << prefix << ";" << endl; + } else if (ttype->is_set()) { + string iter = tmp("_iter"); + indent(out) << "Hashtbl.iter (fun "< fun _ -> "; + indent_up(); + generate_serialize_set_element(out, (t_set*)ttype, iter); + indent_down(); + indent(out) << ") " << prefix << ";" << endl; + } else if (ttype->is_list()) { + string iter = tmp("_iter"); + indent(out) << "List.iter (fun "< "; + indent_up(); + generate_serialize_list_element(out, (t_list*)ttype, iter); + indent_down(); + indent(out) << ") " << prefix << ";" << endl; + } + + if (ttype->is_map()) { + indent(out) << + "oprot#writeMapEnd"; + } else if (ttype->is_set()) { + indent(out) << + "oprot#writeSetEnd"; + } else if (ttype->is_list()) { + indent(out) << + "oprot#writeListEnd"; + } +} + +/** + * Serializes the members of a map. + * + */ +void t_ocaml_generator::generate_serialize_map_element(ofstream &out, + t_map* tmap, + string kiter, + string viter) { + t_field kfield(tmap->get_key_type(), kiter); + generate_serialize_field(out, &kfield); + + t_field vfield(tmap->get_val_type(), viter); + generate_serialize_field(out, &vfield); +} + +/** + * Serializes the members of a set. + */ +void t_ocaml_generator::generate_serialize_set_element(ofstream &out, + t_set* tset, + string iter) { + t_field efield(tset->get_elem_type(), iter); + generate_serialize_field(out, &efield); +} + +/** + * Serializes the members of a list. + */ +void t_ocaml_generator::generate_serialize_list_element(ofstream &out, + t_list* tlist, + string iter) { + t_field efield(tlist->get_elem_type(), iter); + generate_serialize_field(out, &efield); +} + + + +/** + * Renders a function signature of the form 'name args' + * + * @param tfunction Function definition + * @return String of rendered function definition + */ +string t_ocaml_generator::function_signature(t_function* tfunction, + string prefix) { + return + prefix + decapitalize(tfunction->get_name()) + + " " + argument_list(tfunction->get_arglist()); +} + +string t_ocaml_generator::function_type(t_function* tfunc, bool method, bool options){ + string result=""; + + const vector& fields = tfunc->get_arglist()->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + result += render_ocaml_type((*f_iter)->get_type()); + if(options) + result += " option"; + result += " -> "; + } + if(fields.empty() && !method){ + result += "unit -> "; + } + result += render_ocaml_type(tfunc->get_returntype()); + return result; +} + +/** + * Renders a field list + */ +string t_ocaml_generator::argument_list(t_struct* tstruct) { + string result = ""; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + result += " "; + } + result += (*f_iter)->get_name(); + } + return result; +} + +string t_ocaml_generator::type_name(t_type* ttype) { + string prefix = ""; + t_program* program = ttype->get_program(); + if (program != NULL && program != program_) { + if (!ttype->is_service()) { + prefix = capitalize(program->get_name()) + "_types."; + } + } + + string name = ttype->get_name(); + if(ttype->is_service()){ + name = capitalize(name); + } else { + name = decapitalize(name); + } + return prefix + name; +} + +/** + * Converts the parse type to a Protocol.t_type enum + */ +string t_ocaml_generator::type_to_enum(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + return "Protocol.T_VOID"; + case t_base_type::TYPE_STRING: + return "Protocol.T_STRING"; + case t_base_type::TYPE_BOOL: + return "Protocol.T_BOOL"; + case t_base_type::TYPE_BYTE: + return "Protocol.T_BYTE"; + case t_base_type::TYPE_I16: + return "Protocol.T_I16"; + case t_base_type::TYPE_I32: + return "Protocol.T_I32"; + case t_base_type::TYPE_I64: + return "Protocol.T_I64"; + case t_base_type::TYPE_DOUBLE: + return "Protocol.T_DOUBLE"; + } + } else if (type->is_enum()) { + return "Protocol.T_I32"; + } else if (type->is_struct() || type->is_xception()) { + return "Protocol.T_STRUCT"; + } else if (type->is_map()) { + return "Protocol.T_MAP"; + } else if (type->is_set()) { + return "Protocol.T_SET"; + } else if (type->is_list()) { + return "Protocol.T_LIST"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + +/** + * Converts the parse type to an ocaml type + */ +string t_ocaml_generator::render_ocaml_type(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + return "unit"; + case t_base_type::TYPE_STRING: + return "string"; + case t_base_type::TYPE_BOOL: + return "bool"; + case t_base_type::TYPE_BYTE: + return "int"; + case t_base_type::TYPE_I16: + return "int"; + case t_base_type::TYPE_I32: + return "int"; + case t_base_type::TYPE_I64: + return "Int64.t"; + case t_base_type::TYPE_DOUBLE: + return "float"; + } + } else if (type->is_enum()) { + return capitalize(((t_enum*)type)->get_name())+".t"; + } else if (type->is_struct() || type->is_xception()) { + return type_name((t_struct*)type); + } else if (type->is_map()) { + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + return "("+render_ocaml_type(ktype)+","+render_ocaml_type(vtype)+") Hashtbl.t"; + } else if (type->is_set()) { + t_type* etype = ((t_set*)type)->get_elem_type(); + return "("+render_ocaml_type(etype)+",bool) Hashtbl.t"; + } else if (type->is_list()) { + t_type* etype = ((t_list*)type)->get_elem_type(); + return render_ocaml_type(etype)+" list"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + + +THRIFT_REGISTER_GENERATOR(ocaml, "OCaml", ""); diff --git a/compiler/cpp/src/generate/t_oop_generator.h b/compiler/cpp/src/generate/t_oop_generator.h new file mode 100644 index 00000000..bf757862 --- /dev/null +++ b/compiler/cpp/src/generate/t_oop_generator.h @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_OOP_GENERATOR_H +#define T_OOP_GENERATOR_H + +#include +#include + +#include "globals.h" +#include "t_generator.h" + +#include + +/** + * Class with utility methods shared across common object oriented languages. + * Specifically, most of this stuff is for C++/Java. + * + */ +class t_oop_generator : public t_generator { + public: + t_oop_generator(t_program* program) : + t_generator(program) {} + + /** + * Scoping, using curly braces! + */ + + void scope_up(std::ostream& out) { + indent(out) << "{" << std::endl; + indent_up(); + } + + void scope_down(std::ostream& out) { + indent_down(); + indent(out) << "}" << std::endl; + } + + std::string upcase_string(std::string original) { + std::transform(original.begin(), original.end(), original.begin(), (int(*)(int)) toupper); + return original; + } + + /** + * Generates a comment about this code being autogenerated, using C++ style + * comments, which are also fair game in Java / PHP, yay! + * + * @return C-style comment mentioning that this file is autogenerated. + */ + virtual std::string autogen_comment() { + return + std::string("/**\n") + + " * Autogenerated by Thrift\n" + + " *\n" + + " * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n" + + " */\n"; + } +}; + +#endif + diff --git a/compiler/cpp/src/generate/t_perl_generator.cc b/compiler/cpp/src/generate/t_perl_generator.cc new file mode 100644 index 00000000..ae204fd9 --- /dev/null +++ b/compiler/cpp/src/generate/t_perl_generator.cc @@ -0,0 +1,1815 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include "t_oop_generator.h" +#include "platform.h" +using namespace std; + + +/** + * PERL code generator. + * + */ +class t_perl_generator : public t_oop_generator { + public: + t_perl_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_oop_generator(program) + { + out_dir_base_ = "gen-perl"; + escape_['$'] = "\\$"; + escape_['@'] = "\\@"; + } + + /** + * Init and close methods + */ + + void init_generator(); + void close_generator(); + + /** + * Program-level generation functions + */ + + void generate_typedef (t_typedef* ttypedef); + void generate_enum (t_enum* tenum); + void generate_const (t_const* tconst); + void generate_struct (t_struct* tstruct); + void generate_xception (t_struct* txception); + void generate_service (t_service* tservice); + + std::string render_const_value(t_type* type, t_const_value* value); + + /** + * Structs! + */ + + void generate_perl_struct(t_struct* tstruct, bool is_exception); + void generate_perl_struct_definition(std::ofstream& out, t_struct* tstruct, bool is_xception=false); + void generate_perl_struct_reader(std::ofstream& out, t_struct* tstruct); + void generate_perl_struct_writer(std::ofstream& out, t_struct* tstruct); + void generate_perl_function_helpers(t_function* tfunction); + + /** + * Service-level generation functions + */ + + void generate_service_helpers (t_service* tservice); + void generate_service_interface (t_service* tservice); + void generate_service_rest (t_service* tservice); + void generate_service_client (t_service* tservice); + void generate_service_processor (t_service* tservice); + void generate_process_function (t_service* tservice, t_function* tfunction); + + /** + * Serialization constructs + */ + + void generate_deserialize_field (std::ofstream &out, + t_field* tfield, + std::string prefix="", + bool inclass=false); + + void generate_deserialize_struct (std::ofstream &out, + t_struct* tstruct, + std::string prefix=""); + + void generate_deserialize_container (std::ofstream &out, + t_type* ttype, + std::string prefix=""); + + void generate_deserialize_set_element (std::ofstream &out, + t_set* tset, + std::string prefix=""); + + void generate_deserialize_map_element (std::ofstream &out, + t_map* tmap, + std::string prefix=""); + + void generate_deserialize_list_element (std::ofstream &out, + t_list* tlist, + std::string prefix=""); + + void generate_serialize_field (std::ofstream &out, + t_field* tfield, + std::string prefix=""); + + void generate_serialize_struct (std::ofstream &out, + t_struct* tstruct, + std::string prefix=""); + + void generate_serialize_container (std::ofstream &out, + t_type* ttype, + std::string prefix=""); + + void generate_serialize_map_element (std::ofstream &out, + t_map* tmap, + std::string kiter, + std::string viter); + + void generate_serialize_set_element (std::ofstream &out, + t_set* tmap, + std::string iter); + + void generate_serialize_list_element (std::ofstream &out, + t_list* tlist, + std::string iter); + + /** + * Helper rendering functions + */ + + std::string perl_includes(); + std::string declare_field(t_field* tfield, bool init=false, bool obj=false); + std::string function_signature(t_function* tfunction, std::string prefix=""); + std::string argument_list(t_struct* tstruct); + std::string type_to_enum(t_type* ttype); + + std::string autogen_comment() { + return + std::string("#\n") + + "# Autogenerated by Thrift\n" + + "#\n" + + "# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n" + + "#\n"; + } + + void perl_namespace_dirs(t_program* p, std::list& dirs) { + std::string ns = p->get_namespace("perl"); + std::string::size_type loc; + + if (ns.size() > 0) { + while ((loc = ns.find(".")) != std::string::npos) { + dirs.push_back(ns.substr(0, loc)); + ns = ns.substr(loc+1); + } + } + + if (ns.size() > 0) { + dirs.push_back(ns); + } + } + + std::string perl_namespace(t_program* p) { + std::string ns = p->get_namespace("perl"); + std::string result = ""; + std::string::size_type loc; + + if (ns.size() > 0) { + while ((loc = ns.find(".")) != std::string::npos) { + result += ns.substr(0, loc); + result += "::"; + ns = ns.substr(loc+1); + } + + if (ns.size() > 0) { + result += ns + "::"; + } + } + + return result; + } + + std::string get_namespace_out_dir() { + std::string outdir = get_out_dir(); + std::list dirs; + perl_namespace_dirs(program_, dirs); + std::list::iterator it; + for (it = dirs.begin(); it != dirs.end(); it++) { + outdir += *it + "/"; + } + return outdir; + } + + private: + + /** + * File streams + */ + std::ofstream f_types_; + std::ofstream f_consts_; + std::ofstream f_helpers_; + std::ofstream f_service_; + +}; + + +/** + * Prepares for file generation by opening up the necessary file output + * streams. + * + * @param tprogram The program to generate + */ +void t_perl_generator::init_generator() { + // Make output directory + MKDIR(get_out_dir().c_str()); + + string outdir = get_out_dir(); + std::list dirs; + perl_namespace_dirs(program_, dirs); + std::list::iterator it; + for (it = dirs.begin(); it != dirs.end(); it++) { + outdir += *it + "/"; + MKDIR(outdir.c_str()); + } + + // Make output file + string f_types_name = outdir+"Types.pm"; + f_types_.open(f_types_name.c_str()); + string f_consts_name = outdir+"Constants.pm"; + f_consts_.open(f_consts_name.c_str()); + + // Print header + f_types_ << + autogen_comment() << + perl_includes(); + + // Print header + f_consts_ << + autogen_comment() << + "package "<< perl_namespace(program_) <<"Constants;"<get_name()<<";"< constants = tenum->get_constants(); + vector::iterator c_iter; + int value = -1; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + + f_types_ << "use constant "<<(*c_iter)->get_name() << " => " << value << ";" << endl; + } +} + +/** + * Generate a constant value + */ +void t_perl_generator::generate_const(t_const* tconst) { + t_type* type = tconst->get_type(); + string name = tconst->get_name(); + t_const_value* value = tconst->get_value(); + + f_consts_ << "use constant " << name << " => "; + f_consts_ << render_const_value(type, value); + f_consts_ << ";" << endl << endl; +} + +/** + * Prints the value of a constant with the given type. Note that type checking + * is NOT performed in this function as it is always run beforehand using the + * validate_types method in main.cc + */ +string t_perl_generator::render_const_value(t_type* type, t_const_value* value) { + std::ostringstream out; + + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + out << '"' << get_escaped_string(value) << '"'; + break; + case t_base_type::TYPE_BOOL: + out << (value->get_integer() > 0 ? "1" : "0"); + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + out << value->get_integer(); + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() == t_const_value::CV_INTEGER) { + out << value->get_integer(); + } else { + out << value->get_double(); + } + break; + default: + throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + out << value->get_integer(); + } else if (type->is_struct() || type->is_xception()) { + out << "new " << perl_namespace(type->get_program()) << type->get_name() << "({" << endl; + indent_up(); + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + out << render_const_value(g_type_string, v_iter->first); + out << " => "; + out << render_const_value(field_type, v_iter->second); + out << ","; + out << endl; + } + + out << "})"; + } else if (type->is_map()) { + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + out << "{" << endl; + + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + out << render_const_value(ktype, v_iter->first); + out << " => "; + out << render_const_value(vtype, v_iter->second); + out << "," << endl; + } + + out << "}"; + } else if (type->is_list() || type->is_set()) { + t_type* etype; + if (type->is_list()) { + etype = ((t_list*)type)->get_elem_type(); + } else { + etype = ((t_set*)type)->get_elem_type(); + } + out << "[" << endl; + const vector& val = value->get_list(); + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + + out << render_const_value(etype, *v_iter); + if (type->is_set()) { + out << " => 1"; + } + out << "," << endl; + } + out << "]"; + } + return out.str(); +} + +/** + * Make a struct + */ +void t_perl_generator::generate_struct(t_struct* tstruct) { + generate_perl_struct(tstruct, false); +} + +/** + * Generates a struct definition for a thrift exception. Basically the same + * as a struct but extends the Exception class. + * + * @param txception The struct definition + */ +void t_perl_generator::generate_xception(t_struct* txception) { + generate_perl_struct(txception, true); +} + +/** + * Structs can be normal or exceptions. + */ +void t_perl_generator::generate_perl_struct(t_struct* tstruct, + bool is_exception) { + generate_perl_struct_definition(f_types_, tstruct, is_exception); +} + +/** + * Generates a struct definition for a thrift data type. This is nothing in PERL + * where the objects are all just associative arrays (unless of course we + * decide to start using objects for them...) + * + * @param tstruct The struct definition + */ +void t_perl_generator::generate_perl_struct_definition(ofstream& out, + t_struct* tstruct, + bool is_exception) { + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + + out << + "package " << perl_namespace(tstruct->get_program()) << tstruct->get_name() <<";\n"; + if (is_exception) { + out << "use base('Thrift::TException');\n"; + } + + //Create simple acessor methods + out << "use Class::Accessor;\n"; + out << "use base('Class::Accessor');\n"; + + if (members.size() > 0) { + out << perl_namespace(tstruct->get_program()) << tstruct->get_name() <<"->mk_accessors( qw( "; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + t_type* t = get_true_type((*m_iter)->get_type()); + if (!t->is_xception()) { + out << (*m_iter)->get_name() << " "; + } + } + + out << ") );\n"; + } + + + // new() + out << "sub new {\n"; + indent_up(); + out << "my $classname = shift;\n"; + out << "my $self = {};\n"; + out << "my $vals = shift || {};\n"; + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + string dval = "undef"; + t_type* t = get_true_type((*m_iter)->get_type()); + if ((*m_iter)->get_value() != NULL && !(t->is_struct() || t->is_xception())) { + dval = render_const_value((*m_iter)->get_type(), (*m_iter)->get_value()); + } + out << + "$self->{" << (*m_iter)->get_name() << "} = " << dval << ";" << endl; + } + + // Generate constructor from array + if (members.size() > 0) { + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + t_type* t = get_true_type((*m_iter)->get_type()); + if ((*m_iter)->get_value() != NULL && (t->is_struct() || t->is_xception())) { + indent(out) << "$self->{" << (*m_iter)->get_name() << "} = " << render_const_value(t, (*m_iter)->get_value()) << ";" << endl; + } + } + + out << indent() << "if (UNIVERSAL::isa($vals,'HASH')) {" << endl; + indent_up(); + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + out << + indent() << "if (defined $vals->{" << (*m_iter)->get_name() << "}) {" << endl << + indent() << " $self->{" << (*m_iter)->get_name() << "} = $vals->{" << (*m_iter)->get_name() << "};" << endl << + indent() << "}" << endl; + } + indent_down(); + out << + indent() << "}" << endl; + + } + + out << "return bless($self,$classname);\n"; + indent_down(); + out << "}\n\n"; + + out << + "sub getName {" << endl << + indent() << " return '" << tstruct->get_name() << "';" << endl << + indent() << "}" << endl << + endl; + + generate_perl_struct_reader(out, tstruct); + generate_perl_struct_writer(out, tstruct); + +} + +/** + * Generates the read() method for a struct + */ +void t_perl_generator::generate_perl_struct_reader(ofstream& out, + t_struct* tstruct) { + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + out << "sub read {" <readStructBegin(\\$fname);" << endl; + + + // Loop over reading in fields + indent(out) << "while (1) " << endl; + + scope_up(out); + + indent(out) << "$xfer += $input->readFieldBegin(\\$fname, \\$ftype, \\$fid);" << endl; + + // Check for field STOP marker and break + indent(out) << "if ($ftype == TType::STOP) {" << endl; + indent_up(); + indent(out) << "last;" << endl; + indent_down(); + indent(out) << "}" << endl; + + // Switch statement on the field we are reading + indent(out) << "SWITCH: for($fid)" << endl; + + scope_up(out); + + // Generate deserialization code for known cases + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + + indent(out) << "/^" << (*f_iter)->get_key() << "$/ && do{"; + indent(out) << "if ($ftype == " << type_to_enum((*f_iter)->get_type()) << ") {" << endl; + + indent_up(); + generate_deserialize_field(out, *f_iter, "self->"); + indent_down(); + + indent(out) << "} else {" << endl; + + indent(out) << " $xfer += $input->skip($ftype);" << endl; + + out << + indent() << "}" << endl << + indent() << "last; };" << endl; + + } + // In the default case we skip the field + + indent(out) << " $xfer += $input->skip($ftype);" << endl; + + scope_down(out); + + indent(out) << "$xfer += $input->readFieldEnd();" << endl; + + scope_down(out); + + indent(out) << "$xfer += $input->readStructEnd();" << endl; + + indent(out) << "return $xfer;" << endl; + + indent_down(); + out << indent() << "}" << endl << endl; +} + +/** + * Generates the write() method for a struct + */ +void t_perl_generator::generate_perl_struct_writer(ofstream& out, + t_struct* tstruct) { + string name = tstruct->get_name(); + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + out << "sub write {" << endl; + + indent_up(); + indent(out) << "my $self = shift;"<writeStructBegin('" << name << "');" << endl; + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + out << indent() << "if (defined $self->{" << (*f_iter)->get_name() << "}) {" << endl; + indent_up(); + + indent(out) << + "$xfer += $output->writeFieldBegin(" << + "'" << (*f_iter)->get_name() << "', " << + type_to_enum((*f_iter)->get_type()) << ", " << + (*f_iter)->get_key() << ");" << endl; + + + // Write field contents + generate_serialize_field(out, *f_iter, "self->"); + + indent(out) << + "$xfer += $output->writeFieldEnd();" << endl; + + indent_down(); + indent(out) << "}" << endl; + } + + + out << + indent() << "$xfer += $output->writeFieldStop();" << endl << + indent() << "$xfer += $output->writeStructEnd();" << endl; + + out <get_program()) << "Types;" << endl; + + t_service* extends_s = tservice->get_extends(); + if (extends_s != NULL) { + f_service_ << + "use " << perl_namespace(extends_s->get_program()) << extends_s->get_name() << ";" << endl; + } + + f_service_ << + endl; + + // Generate the three main parts of the service (well, two for now in PERL) + generate_service_helpers(tservice); + generate_service_interface(tservice); + generate_service_rest(tservice); + generate_service_client(tservice); + generate_service_processor(tservice); + + // Close service file + f_service_ << "1;" << endl; + f_service_.close(); +} + +/** + * Generates a service server definition. + * + * @param tservice The service to generate a server for. + */ +void t_perl_generator::generate_service_processor(t_service* tservice) { + // Generate the dispatch methods + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + string extends = ""; + string extends_processor = ""; + t_service* extends_s = tservice->get_extends(); + if (extends_s != NULL) { + extends = perl_namespace(extends_s->get_program()) + extends_s->get_name(); + extends_processor = "use base('" + extends + "Processor');"; + } + + indent_up(); + + // Generate the header portion + f_service_ << + "package " << perl_namespace(program_) << service_name_ << "Processor;" << endl << extends_processor << endl; + + + if (extends.empty()) { + f_service_ << "sub new {" << endl; + + indent_up(); + + f_service_ << + indent() << "my $classname = shift;"<< endl << + indent() << "my $handler = shift;"<< endl << + indent() << "my $self = {};" << endl; + + f_service_ << + indent() << "$self->{handler} = $handler;" << endl; + + f_service_ << + indent() << "return bless($self,$classname);"<readMessageBegin(\\$fname, \\$mtype, \\$rseqid);" << endl; + + // HOT: check for method implementation + f_service_ << + indent() << "my $methodname = 'process_'.$fname;" << endl << + indent() << "if (!method_exists($self, $methodname)) {" << endl; + + f_service_ << + indent() << " $input->skip(TType::STRUCT);" << endl << + indent() << " $input->readMessageEnd();" << endl << + indent() << " my $x = new TApplicationException('Function '.$fname.' not implemented.', TApplicationException::UNKNOWN_METHOD);" << endl << + indent() << " $output->writeMessageBegin($fname, TMessageType::EXCEPTION, $rseqid);" << endl << + indent() << " $x->write($output);" << endl << + indent() << " $output->writeMessageEnd();" << endl << + indent() << " $output->getTransport()->flush();" << endl << + indent() << " return;" << endl; + + f_service_ << + indent() << "}" << endl << + indent() << "$self->$methodname($rseqid, $input, $output);" << endl << + indent() << "return 1;" << endl; + + indent_down(); + + f_service_ << + indent() << "}" << endl <get_name() << "{"<get_program()) + service_name_ + "_" + tfunction->get_name() + "_args"; + string resultname = perl_namespace(tservice->get_program()) + service_name_ + "_" + tfunction->get_name() + "_result"; + + f_service_ << + indent() << "my $args = new " << argsname << "();" << endl << + indent() << "$args->read($input);" << endl; + + f_service_ << + indent() << "$input->readMessageEnd();" << endl; + + t_struct* xs = tfunction->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + + // Declare result for non oneway function + if (!tfunction->is_oneway()) { + f_service_ << + indent() << "my $result = new " << resultname << "();" << endl; + } + + // Try block for a function with exceptions + if (xceptions.size() > 0) { + f_service_ << + indent() << "eval {" << endl; + indent_up(); + } + + // Generate the function call + t_struct* arg_struct = tfunction->get_arglist(); + const std::vector& fields = arg_struct->get_members(); + vector::const_iterator f_iter; + + f_service_ << indent(); + if (!tfunction->is_oneway() && !tfunction->get_returntype()->is_void()) { + f_service_ << "$result->{success} = "; + } + f_service_ << + "$self->{handler}->" << tfunction->get_name() << "("; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << "$args->" << (*f_iter)->get_name(); + } + f_service_ << ");" << endl; + + if (!tfunction->is_oneway() && xceptions.size() > 0) { + indent_down(); + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << + indent() << "}; if( UNIVERSAL::isa($@,'"<<(*x_iter)->get_type()->get_name()<<"') ){ "<is_oneway()) { + indent_up(); + f_service_ << + indent() << "$result->{" << (*x_iter)->get_name() << "} = $@;" << endl; + indent_down(); + f_service_ << indent(); + } + } + indent_down(); + f_service_ << "}" << endl; + } + + // Shortcut out here for oneway functions + if (tfunction->is_oneway()) { + f_service_ << + indent() << "return;" << endl; + indent_down(); + f_service_ << + indent() << "}" << endl; + return; + } + indent_up(); + // Serialize the request header + f_service_ << + indent() << "$output->writeMessageBegin('" << tfunction->get_name() << "', TMessageType::REPLY, $seqid);" << endl << + indent() << "$result->write($output);" << endl << + indent() << "$output->getTransport()->flush();" << endl; + indent_down(); + + // Close function + indent_down(); + f_service_ << + indent() << "}" << endl; +} + +/** + * Generates helper functions for a service. + * + * @param tservice The service to generate a header definition for + */ +void t_perl_generator::generate_service_helpers(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + f_service_ << + "# HELPER FUNCTIONS AND STRUCTURES" << endl << endl; + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* ts = (*f_iter)->get_arglist(); + string name = ts->get_name(); + ts->set_name(service_name_ + "_" + name); + generate_perl_struct_definition(f_service_, ts, false); + generate_perl_function_helpers(*f_iter); + ts->set_name(name); + } +} + +/** + * Generates a struct and helpers for a function. + * + * @param tfunction The function + */ +void t_perl_generator::generate_perl_function_helpers(t_function* tfunction) { + t_struct result(program_, service_name_ + "_" + tfunction->get_name() + "_result"); + t_field success(tfunction->get_returntype(), "success", 0); + if (!tfunction->get_returntype()->is_void()) { + result.append(&success); + } + + t_struct* xs = tfunction->get_xceptions(); + const vector& fields = xs->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + result.append(*f_iter); + } + + generate_perl_struct_definition(f_service_, &result, false); +} + +/** + * Generates a service interface definition. + * + * @param tservice The service to generate a header definition for + */ +void t_perl_generator::generate_service_interface(t_service* tservice) { + string extends_if = ""; + t_service* extends_s = tservice->get_extends(); + if (extends_s != NULL) { + extends_if = "use base('" + perl_namespace(extends_s->get_program()) + extends_s->get_name() + "If');"; + } + + f_service_ << + "package " << perl_namespace(program_) << service_name_ << "If;"< functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_service_ << + "sub " << function_signature(*f_iter) <get_extends(); + if (extends_s != NULL) { + extends = extends_s->get_name(); + extends_if = "use base('" + perl_namespace(extends_s->get_program()) + extends_s->get_name() + "Rest');"; + } + f_service_ << + "package " << perl_namespace(program_) << service_name_ << "Rest;"< $impl };"< functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_service_ << + "sub " << (*f_iter)->get_name() << + "{" <& args = (*f_iter)->get_arglist()->get_members(); + vector::const_iterator a_iter; + for (a_iter = args.begin(); a_iter != args.end(); ++a_iter) { + t_type* atype = get_true_type((*a_iter)->get_type()); + string req = "$request->{'" + (*a_iter)->get_name() + "'}"; + f_service_ << + indent() << "my $" << (*a_iter)->get_name() << " = (" << req << ") ? " << req << " : undef;" << endl; + if (atype->is_string() && + ((t_base_type*)atype)->is_string_list()) { + f_service_ << + indent() << "my @" << (*a_iter)->get_name() << " = split(/,/, $" << (*a_iter)->get_name() << ");" << endl << + indent() << "$"<<(*a_iter)->get_name() <<" = \\@"<<(*a_iter)->get_name()<{impl}->" << (*f_iter)->get_name() << "(" << argument_list((*f_iter)->get_arglist()) << ");" << endl; + indent_down(); + indent(f_service_) << "}" << endl <get_extends(); + if (extends_s != NULL) { + extends = perl_namespace(extends_s->get_program()) + extends_s->get_name(); + extends_client = "use base('" + extends + "Client');"; + } + + f_service_ << + "package " << perl_namespace(program_) << service_name_ << "Client;"<SUPER::new($input, $output);" << endl; + } else { + f_service_ << + indent() << " $self->{input} = $input;" << endl << + indent() << " $self->{output} = defined $output ? $output : $input;" << endl << + indent() << " $self->{seqid} = 0;" << endl; + } + + f_service_ << + indent() << "return bless($self,$classname);"< functions = tservice->get_functions(); + vector::const_iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* arg_struct = (*f_iter)->get_arglist(); + const vector& fields = arg_struct->get_members(); + vector::const_iterator fld_iter; + string funname = (*f_iter)->get_name(); + + // Open function + f_service_ << "sub " << function_signature(*f_iter) << endl; + + indent_up(); + + indent(f_service_) << indent() << + "$self->send_" << funname << "("; + + bool first = true; + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << "$" << (*fld_iter)->get_name(); + } + f_service_ << ");" << endl; + + if (!(*f_iter)->is_oneway()) { + f_service_ << indent(); + if (!(*f_iter)->get_returntype()->is_void()) { + f_service_ << "return "; + } + f_service_ << + "$self->recv_" << funname << "();" << endl; + } + + indent_down(); + + f_service_ << "}" << endl << endl; + + f_service_ << + "sub send_" << function_signature(*f_iter) << endl; + + indent_up(); + + std::string argsname = perl_namespace(tservice->get_program()) + service_name_ + "_" + (*f_iter)->get_name() + "_args"; + + // Serialize the request header + f_service_ << + indent() << "$self->{output}->writeMessageBegin('" << (*f_iter)->get_name() << "', TMessageType::CALL, $self->{seqid});" << endl; + + f_service_ << + indent() << "my $args = new " << argsname << "();" << endl; + + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + f_service_ << + indent() << "$args->{" << (*fld_iter)->get_name() << "} = $" << (*fld_iter)->get_name() << ";" << endl; + } + + // Write to the stream + f_service_ << + indent() << "$args->write($self->{output});" << endl << + indent() << "$self->{output}->writeMessageEnd();" << endl << + indent() << "$self->{output}->getTransport()->flush();" << endl; + + + indent_down(); + + f_service_ << "}" << endl; + + + if (!(*f_iter)->is_oneway()) { + std::string resultname = perl_namespace(tservice->get_program()) + service_name_ + "_" + (*f_iter)->get_name() + "_result"; + t_struct noargs(program_); + + t_function recv_function((*f_iter)->get_returntype(), + string("recv_") + (*f_iter)->get_name(), + &noargs); + // Open function + f_service_ << + endl << + "sub " << function_signature(&recv_function) << endl; + + indent_up(); + + f_service_ << + indent() << "my $rseqid = 0;" << endl << + indent() << "my $fname;" << endl << + indent() << "my $mtype = 0;" << endl << + endl; + + f_service_ << + indent() << "$self->{input}->readMessageBegin(\\$fname, \\$mtype, \\$rseqid);" << endl << + indent() << "if ($mtype == TMessageType::EXCEPTION) {" << endl << + indent() << " my $x = new TApplicationException();" << endl << + indent() << " $x->read($self->{input});" << endl << + indent() << " $self->{input}->readMessageEnd();" << endl << + indent() << " die $x;" << endl << + indent() << "}" << endl; + + + f_service_ << + indent() << "my $result = new " << resultname << "();" << endl << + indent() << "$result->read($self->{input});" << endl; + + + f_service_ << + indent() << "$self->{input}->readMessageEnd();" << endl << + endl; + + + // Careful, only return result if not a void function + if (!(*f_iter)->get_returntype()->is_void()) { + f_service_ << + indent() << "if (defined $result->{success} ) {" << endl << + indent() << " return $result->{success};" << endl << + indent() << "}" << endl; + } + + t_struct* xs = (*f_iter)->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << + indent() << "if (defined $result->{" << (*x_iter)->get_name() << "}) {" << endl << + indent() << " die $result->{" << (*x_iter)->get_name() << "};" << endl << + indent() << "}" << endl; + } + + // Careful, only return _result if not a void function + if ((*f_iter)->get_returntype()->is_void()) { + indent(f_service_) << + "return;" << endl; + } else { + f_service_ << + indent() << "die \"" << (*f_iter)->get_name() << " failed: unknown result\";" << endl; + } + + // Close function + indent_down(); + f_service_ << "}"<get_type()); + + if (type->is_void()) { + throw "CANNOT GENERATE DESERIALIZE CODE FOR void TYPE: " + + prefix + tfield->get_name(); + } + + string name = tfield->get_name(); + + //Hack for when prefix is defined (always a hash ref) + if (!prefix.empty()) { + name = prefix + "{" + tfield->get_name() + "}"; + } + + if (type->is_struct() || type->is_xception()) { + generate_deserialize_struct(out, + (t_struct*)type, + name); + } else if (type->is_container()) { + generate_deserialize_container(out, type, name); + } else if (type->is_base_type() || type->is_enum()) { + indent(out) << + "$xfer += $input->"; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "compiler error: cannot serialize void field in a struct: " + + name; + break; + case t_base_type::TYPE_STRING: + out << "readString(\\$" << name << ");"; + break; + case t_base_type::TYPE_BOOL: + out << "readBool(\\$" << name << ");"; + break; + case t_base_type::TYPE_BYTE: + out << "readByte(\\$" << name << ");"; + break; + case t_base_type::TYPE_I16: + out << "readI16(\\$" << name << ");"; + break; + case t_base_type::TYPE_I32: + out << "readI32(\\$" << name << ");"; + break; + case t_base_type::TYPE_I64: + out << "readI64(\\$" << name << ");"; + break; + case t_base_type::TYPE_DOUBLE: + out << "readDouble(\\$" << name << ");"; + break; + default: + throw "compiler error: no PERL name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + out << "readI32(\\$" << name << ");"; + } + out << endl; + + } else { + printf("DO NOT KNOW HOW TO DESERIALIZE FIELD '%s' TYPE '%s'\n", + tfield->get_name().c_str(), type->get_name().c_str()); + } +} + +/** + * Generates an unserializer for a variable. This makes two key assumptions, + * first that there is a const char* variable named data that points to the + * buffer for deserialization, and that there is a variable protocol which + * is a reference to a TProtocol serialization object. + */ +void t_perl_generator::generate_deserialize_struct(ofstream &out, + t_struct* tstruct, + string prefix) { + out << + indent() << "$" << prefix << " = new " << perl_namespace(tstruct->get_program()) << tstruct->get_name() << "();" << endl << + indent() << "$xfer += $" << prefix << "->read($input);" << endl; +} + +void t_perl_generator::generate_deserialize_container(ofstream &out, + t_type* ttype, + string prefix) { + scope_up(out); + + string size = tmp("_size"); + string ktype = tmp("_ktype"); + string vtype = tmp("_vtype"); + string etype = tmp("_etype"); + + t_field fsize(g_type_i32, size); + t_field fktype(g_type_byte, ktype); + t_field fvtype(g_type_byte, vtype); + t_field fetype(g_type_byte, etype); + + out << + indent() << "my $" << size << " = 0;" << endl; + + // Declare variables, read header + if (ttype->is_map()) { + out << + indent() << "$" << prefix << " = {};" << endl << + indent() << "my $" << ktype << " = 0;" << endl << + indent() << "my $" << vtype << " = 0;" << endl; + + out << + indent() << "$xfer += $input->readMapBegin(" << + "\\$" << ktype << ", \\$" << vtype << ", \\$" << size << ");" << endl; + + } else if (ttype->is_set()) { + + out << + indent() << "$" << prefix << " = {};" << endl << + indent() << "my $" << etype << " = 0;" << endl << + indent() << "$xfer += $input->readSetBegin(" << + "\\$" << etype << ", \\$" << size << ");" << endl; + + } else if (ttype->is_list()) { + + out << + indent() << "$" << prefix << " = [];" << endl << + indent() << "my $" << etype << " = 0;" << endl << + indent() << "$xfer += $input->readListBegin(" << + "\\$" << etype << ", \\$" << size << ");" << endl; + + } + + // For loop iterates over elements + string i = tmp("_i"); + indent(out) << + "for (my $" << + i << " = 0; $" << i << " < $" << size << "; ++$" << i << ")" << endl; + + scope_up(out); + + if (ttype->is_map()) { + generate_deserialize_map_element(out, (t_map*)ttype, prefix); + } else if (ttype->is_set()) { + generate_deserialize_set_element(out, (t_set*)ttype, prefix); + } else if (ttype->is_list()) { + generate_deserialize_list_element(out, (t_list*)ttype, prefix); + } + + scope_down(out); + + + // Read container end + if (ttype->is_map()) { + indent(out) << "$xfer += $input->readMapEnd();" << endl; + } else if (ttype->is_set()) { + indent(out) << "$xfer += $input->readSetEnd();" << endl; + } else if (ttype->is_list()) { + indent(out) << "$xfer += $input->readListEnd();" << endl; + } + + scope_down(out); +} + + +/** + * Generates code to deserialize a map + */ +void t_perl_generator::generate_deserialize_map_element(ofstream &out, + t_map* tmap, + string prefix) { + string key = tmp("key"); + string val = tmp("val"); + t_field fkey(tmap->get_key_type(), key); + t_field fval(tmap->get_val_type(), val); + + indent(out) << + declare_field(&fkey, true, true) << endl; + indent(out) << + declare_field(&fval, true, true) << endl; + + generate_deserialize_field(out, &fkey); + generate_deserialize_field(out, &fval); + + indent(out) << + "$" << prefix << "->{$" << key << "} = $" << val << ";" << endl; +} + +void t_perl_generator::generate_deserialize_set_element(ofstream &out, + t_set* tset, + string prefix) { + string elem = tmp("elem"); + t_field felem(tset->get_elem_type(), elem); + + indent(out) << + "my $" << elem << " = undef;" << endl; + + generate_deserialize_field(out, &felem); + + indent(out) << + "$" << prefix << "->{$" << elem << "} = 1;" << endl; +} + +void t_perl_generator::generate_deserialize_list_element(ofstream &out, + t_list* tlist, + string prefix) { + string elem = tmp("elem"); + t_field felem(tlist->get_elem_type(), elem); + + indent(out) << + "my $" << elem << " = undef;" << endl; + + generate_deserialize_field(out, &felem); + + indent(out) << + "push(@{$" << prefix << "},$" << elem << ");" << endl; +} + + +/** + * Serializes a field of any type. + * + * @param tfield The field to serialize + * @param prefix Name to prepend to field name + */ +void t_perl_generator::generate_serialize_field(ofstream &out, + t_field* tfield, + string prefix) { + t_type* type = get_true_type(tfield->get_type()); + + // Do nothing for void types + if (type->is_void()) { + throw "CANNOT GENERATE SERIALIZE CODE FOR void TYPE: " + + prefix + tfield->get_name(); + } + + if (type->is_struct() || type->is_xception()) { + generate_serialize_struct(out, + (t_struct*)type, + prefix + "{"+tfield->get_name()+"}" ); + } else if (type->is_container()) { + generate_serialize_container(out, + type, + prefix + "{" + tfield->get_name()+"}"); + } else if (type->is_base_type() || type->is_enum()) { + + string name = tfield->get_name(); + + //Hack for when prefix is defined (always a hash ref) + if(!prefix.empty()) + name = prefix + "{" + tfield->get_name() + "}"; + + indent(out) << + "$xfer += $output->"; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw + "compiler error: cannot serialize void field in a struct: " + name; + break; + case t_base_type::TYPE_STRING: + out << "writeString($" << name << ");"; + break; + case t_base_type::TYPE_BOOL: + out << "writeBool($" << name << ");"; + break; + case t_base_type::TYPE_BYTE: + out << "writeByte($" << name << ");"; + break; + case t_base_type::TYPE_I16: + out << "writeI16($" << name << ");"; + break; + case t_base_type::TYPE_I32: + out << "writeI32($" << name << ");"; + break; + case t_base_type::TYPE_I64: + out << "writeI64($" << name << ");"; + break; + case t_base_type::TYPE_DOUBLE: + out << "writeDouble($" << name << ");"; + break; + default: + throw "compiler error: no PERL name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + out << "writeI32($" << name << ");"; + } + out << endl; + + } else { + printf("DO NOT KNOW HOW TO SERIALIZE FIELD '%s%s' TYPE '%s'\n", + prefix.c_str(), + tfield->get_name().c_str(), + type->get_name().c_str()); + } +} + +/** + * Serializes all the members of a struct. + * + * @param tstruct The struct to serialize + * @param prefix String prefix to attach to all fields + */ +void t_perl_generator::generate_serialize_struct(ofstream &out, + t_struct* tstruct, + string prefix) { + indent(out) << + "$xfer += $" << prefix << "->write($output);" << endl; +} + +/** + * Writes out a container + */ +void t_perl_generator::generate_serialize_container(ofstream &out, + t_type* ttype, + string prefix) { + scope_up(out); + + if (ttype->is_map()) { + indent(out) << + "$output->writeMapBegin(" << + type_to_enum(((t_map*)ttype)->get_key_type()) << ", " << + type_to_enum(((t_map*)ttype)->get_val_type()) << ", " << + "scalar(keys %{$" << prefix << "}));" << endl; + } else if (ttype->is_set()) { + indent(out) << + "$output->writeSetBegin(" << + type_to_enum(((t_set*)ttype)->get_elem_type()) << ", " << + "scalar(@{$" << prefix << "}));" << endl; + + } else if (ttype->is_list()) { + + indent(out) << + "$output->writeListBegin(" << + type_to_enum(((t_list*)ttype)->get_elem_type()) << ", " << + "scalar(@{$" << prefix << "}));" << endl; + + } + + scope_up(out); + + if (ttype->is_map()) { + string kiter = tmp("kiter"); + string viter = tmp("viter"); + indent(out) << + "while( my ($"<is_set()) { + string iter = tmp("iter"); + indent(out) << + "foreach my $"<is_list()) { + string iter = tmp("iter"); + indent(out) << + "foreach my $"<is_map()) { + indent(out) << + "$output->writeMapEnd();" << endl; + } else if (ttype->is_set()) { + indent(out) << + "$output->writeSetEnd();" << endl; + } else if (ttype->is_list()) { + indent(out) << + "$output->writeListEnd();" << endl; + } + + scope_down(out); +} + +/** + * Serializes the members of a map. + * + */ +void t_perl_generator::generate_serialize_map_element(ofstream &out, + t_map* tmap, + string kiter, + string viter) { + t_field kfield(tmap->get_key_type(), kiter); + generate_serialize_field(out, &kfield); + + t_field vfield(tmap->get_val_type(), viter); + generate_serialize_field(out, &vfield); +} + +/** + * Serializes the members of a set. + */ +void t_perl_generator::generate_serialize_set_element(ofstream &out, + t_set* tset, + string iter) { + t_field efield(tset->get_elem_type(), iter); + generate_serialize_field(out, &efield); +} + +/** + * Serializes the members of a list. + */ +void t_perl_generator::generate_serialize_list_element(ofstream &out, + t_list* tlist, + string iter) { + t_field efield(tlist->get_elem_type(), iter); + generate_serialize_field(out, &efield); +} + +/** + * Declares a field, which may include initialization as necessary. + * + * @param ttype The type + */ +string t_perl_generator::declare_field(t_field* tfield, bool init, bool obj) { + string result = "my $" + tfield->get_name(); + if (init) { + t_type* type = get_true_type(tfield->get_type()); + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + break; + case t_base_type::TYPE_STRING: + result += " = ''"; + break; + case t_base_type::TYPE_BOOL: + result += " = 0"; + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + result += " = 0"; + break; + case t_base_type::TYPE_DOUBLE: + result += " = 0.0"; + break; + default: + throw "compiler error: no PERL initializer for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + result += " = 0"; + } else if (type->is_container()) { + result += " = []"; + } else if (type->is_struct() || type->is_xception()) { + if (obj) { + result += " = new " + perl_namespace(type->get_program()) + type->get_name() + "()"; + } else { + result += " = undef"; + } + } + } + return result + ";"; +} + +/** + * Renders a function signature of the form 'type name(args)' + * + * @param tfunction Function definition + * @return String of rendered function definition + */ +string t_perl_generator::function_signature(t_function* tfunction, + string prefix) { + + string str; + + str = prefix + tfunction->get_name() + "{\n"; + str += " my $self = shift;\n"; + + //Need to create perl function arg inputs + const vector &fields = tfunction->get_arglist()->get_members(); + vector::const_iterator f_iter; + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + str += " my $" + (*f_iter)->get_name() + " = shift;\n"; + } + + return str; +} + +/** + * Renders a field list + */ +string t_perl_generator::argument_list(t_struct* tstruct) { + string result = ""; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + result += ", "; + } + result += "$" + (*f_iter)->get_name(); + } + return result; +} + +/** + * Converts the parse type to a C++ enum string for the given type. + */ +string t_perl_generator ::type_to_enum(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + return "TType::STRING"; + case t_base_type::TYPE_BOOL: + return "TType::BOOL"; + case t_base_type::TYPE_BYTE: + return "TType::BYTE"; + case t_base_type::TYPE_I16: + return "TType::I16"; + case t_base_type::TYPE_I32: + return "TType::I32"; + case t_base_type::TYPE_I64: + return "TType::I64"; + case t_base_type::TYPE_DOUBLE: + return "TType::DOUBLE"; + } + } else if (type->is_enum()) { + return "TType::I32"; + } else if (type->is_struct() || type->is_xception()) { + return "TType::STRUCT"; + } else if (type->is_map()) { + return "TType::MAP"; + } else if (type->is_set()) { + return "TType::SET"; + } else if (type->is_list()) { + return "TType::LIST"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + +THRIFT_REGISTER_GENERATOR(perl, "Perl", ""); diff --git a/compiler/cpp/src/generate/t_php_generator.cc b/compiler/cpp/src/generate/t_php_generator.cc new file mode 100644 index 00000000..436a6325 --- /dev/null +++ b/compiler/cpp/src/generate/t_php_generator.cc @@ -0,0 +1,2281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include "t_oop_generator.h" +#include "platform.h" +using namespace std; + + +/** + * PHP code generator. + * + */ +class t_php_generator : public t_oop_generator { + public: + t_php_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_oop_generator(program) + { + std::map::const_iterator iter; + + iter = parsed_options.find("inlined"); + binary_inline_ = (iter != parsed_options.end()); + + iter = parsed_options.find("rest"); + rest_ = (iter != parsed_options.end()); + + iter = parsed_options.find("server"); + phps_ = (iter != parsed_options.end()); + + iter = parsed_options.find("autoload"); + autoload_ = (iter != parsed_options.end()); + + iter = parsed_options.find("oop"); + oop_ = (iter != parsed_options.end()); + + if (oop_ && binary_inline_) { + throw "oop and inlined are mutually exclusive."; + } + + out_dir_base_ = (binary_inline_ ? "gen-phpi" : "gen-php"); + escape_['$'] = "\\$"; + } + + /** + * Init and close methods + */ + + void init_generator(); + void close_generator(); + + /** + * Program-level generation functions + */ + + void generate_typedef (t_typedef* ttypedef); + void generate_enum (t_enum* tenum); + void generate_const (t_const* tconst); + void generate_struct (t_struct* tstruct); + void generate_xception (t_struct* txception); + void generate_service (t_service* tservice); + + std::string render_const_value(t_type* type, t_const_value* value); + + /** + * Structs! + */ + + void generate_php_struct(t_struct* tstruct, bool is_exception); + void generate_php_struct_definition(std::ofstream& out, t_struct* tstruct, bool is_xception=false); + void _generate_php_struct_definition(std::ofstream& out, t_struct* tstruct, bool is_xception=false); + void generate_php_struct_reader(std::ofstream& out, t_struct* tstruct); + void generate_php_struct_writer(std::ofstream& out, t_struct* tstruct); + void generate_php_function_helpers(t_function* tfunction); + + void generate_php_type_spec(std::ofstream &out, t_type* t); + void generate_php_struct_spec(std::ofstream &out, t_struct* tstruct); + + /** + * Service-level generation functions + */ + + void generate_service_helpers (t_service* tservice); + void generate_service_interface (t_service* tservice); + void generate_service_rest (t_service* tservice); + void generate_service_client (t_service* tservice); + void _generate_service_client (std::ofstream &out, t_service* tservice); + void generate_service_processor (t_service* tservice); + void generate_process_function (t_service* tservice, t_function* tfunction); + + /** + * Serialization constructs + */ + + void generate_deserialize_field (std::ofstream &out, + t_field* tfield, + std::string prefix="", + bool inclass=false); + + void generate_deserialize_struct (std::ofstream &out, + t_struct* tstruct, + std::string prefix=""); + + void generate_deserialize_container (std::ofstream &out, + t_type* ttype, + std::string prefix=""); + + void generate_deserialize_set_element (std::ofstream &out, + t_set* tset, + std::string prefix=""); + + void generate_deserialize_map_element (std::ofstream &out, + t_map* tmap, + std::string prefix=""); + + void generate_deserialize_list_element (std::ofstream &out, + t_list* tlist, + std::string prefix=""); + + void generate_serialize_field (std::ofstream &out, + t_field* tfield, + std::string prefix=""); + + void generate_serialize_struct (std::ofstream &out, + t_struct* tstruct, + std::string prefix=""); + + void generate_serialize_container (std::ofstream &out, + t_type* ttype, + std::string prefix=""); + + void generate_serialize_map_element (std::ofstream &out, + t_map* tmap, + std::string kiter, + std::string viter); + + void generate_serialize_set_element (std::ofstream &out, + t_set* tmap, + std::string iter); + + void generate_serialize_list_element (std::ofstream &out, + t_list* tlist, + std::string iter); + + /** + * Helper rendering functions + */ + + std::string php_includes(); + std::string declare_field(t_field* tfield, bool init=false, bool obj=false); + std::string function_signature(t_function* tfunction, std::string prefix=""); + std::string argument_list(t_struct* tstruct); + std::string type_to_cast(t_type* ttype); + std::string type_to_enum(t_type* ttype); + + std::string php_namespace(t_program* p) { + std::string ns = p->get_namespace("php"); + return ns.size() ? (ns + "_") : ""; + } + + private: + + /** + * File streams + */ + std::ofstream f_types_; + std::ofstream f_consts_; + std::ofstream f_helpers_; + std::ofstream f_service_; + + /** + * Generate protocol-independent template? Or Binary inline code? + */ + bool binary_inline_; + + /** + * Generate a REST handler class + */ + bool rest_; + + /** + * Generate stubs for a PHP server + */ + bool phps_; + + /** + * Generate PHP code that uses autoload + */ + bool autoload_; + + /** + * Whether to use OOP base class TBase + */ + bool oop_; + +}; + + +/** + * Prepares for file generation by opening up the necessary file output + * streams. + * + * @param tprogram The program to generate + */ +void t_php_generator::init_generator() { + // Make output directory + MKDIR(get_out_dir().c_str()); + + // Make output file + string f_types_name = get_out_dir()+program_name_+"_types.php"; + f_types_.open(f_types_name.c_str()); + + // Print header + f_types_ << + "& includes = program_->get_includes(); + for (size_t i = 0; i < includes.size(); ++i) { + string package = includes[i]->get_name(); + f_types_ << + "include_once $GLOBALS['THRIFT_ROOT'].'/packages/" << package << "/" << package << "_types.php';" << endl; + } + f_types_ << endl; + + // Print header + if (!program_->get_consts().empty()) { + string f_consts_name = get_out_dir()+program_name_+"_constants.php"; + f_consts_.open(f_consts_name.c_str()); + f_consts_ << + "" << endl; + f_types_.close(); + + if (!program_->get_consts().empty()) { + f_consts_ << "?>" << endl; + f_consts_.close(); + } +} + +/** + * Generates a typedef. This is not done in PHP, types are all implicit. + * + * @param ttypedef The type definition + */ +void t_php_generator::generate_typedef(t_typedef* ttypedef) {} + +/** + * Generates code for an enumerated type. Since define is expensive to lookup + * in PHP, we use a global array for this. + * + * @param tenum The enumeration + */ +void t_php_generator::generate_enum(t_enum* tenum) { + f_types_ << + "$GLOBALS['" << php_namespace(tenum->get_program()) << "E_" << tenum->get_name() << "'] = array(" << endl; + + vector constants = tenum->get_constants(); + vector::iterator c_iter; + int value = -1; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + + f_types_ << + " '" << (*c_iter)->get_name() << "' => " << value << "," << endl; + } + + f_types_ << + ");" << endl << endl; + + + // We're also doing it this way to see how it performs. It's more legible + // code but you can't do things like an 'extract' on it, which is a bit of + // a downer. + f_types_ << + "final class " << php_namespace(tenum->get_program()) << tenum->get_name() << " {" << endl; + indent_up(); + + value = -1; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + + indent(f_types_) << + "const " << (*c_iter)->get_name() << " = " << value << ";" << endl; + } + + indent(f_types_) << + "static public $__names = array(" << endl; + value = -1; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + + indent(f_types_) << + " " << value << " => '" << (*c_iter)->get_name() << "'," << endl; + } + indent(f_types_) << + ");" << endl; + + indent_down(); + f_types_ << "}" << endl << endl; +} + +/** + * Generate a constant value + */ +void t_php_generator::generate_const(t_const* tconst) { + t_type* type = tconst->get_type(); + string name = tconst->get_name(); + t_const_value* value = tconst->get_value(); + + f_consts_ << "$GLOBALS['" << program_name_ << "_CONSTANTS']['" << name << "'] = "; + f_consts_ << render_const_value(type, value); + f_consts_ << ";" << endl << endl; +} + +/** + * Prints the value of a constant with the given type. Note that type checking + * is NOT performed in this function as it is always run beforehand using the + * validate_types method in main.cc + */ +string t_php_generator::render_const_value(t_type* type, t_const_value* value) { + std::ostringstream out; + type = get_true_type(type); + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + out << '"' << get_escaped_string(value) << '"'; + break; + case t_base_type::TYPE_BOOL: + out << (value->get_integer() > 0 ? "true" : "false"); + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + out << value->get_integer(); + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() == t_const_value::CV_INTEGER) { + out << value->get_integer(); + } else { + out << value->get_double(); + } + break; + default: + throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + indent(out) << value->get_integer(); + } else if (type->is_struct() || type->is_xception()) { + out << "new " << php_namespace(type->get_program()) << type->get_name() << "(array(" << endl; + indent_up(); + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + out << indent(); + out << render_const_value(g_type_string, v_iter->first); + out << " => "; + out << render_const_value(field_type, v_iter->second); + out << endl; + } + indent_down(); + indent(out) << "))"; + } else if (type->is_map()) { + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + out << "array(" << endl; + indent_up(); + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + out << indent(); + out << render_const_value(ktype, v_iter->first); + out << " => "; + out << render_const_value(vtype, v_iter->second); + out << "," << endl; + } + indent_down(); + indent(out) << ")"; + } else if (type->is_list() || type->is_set()) { + t_type* etype; + if (type->is_list()) { + etype = ((t_list*)type)->get_elem_type(); + } else { + etype = ((t_set*)type)->get_elem_type(); + } + out << "array(" << endl; + indent_up(); + const vector& val = value->get_list(); + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + out << indent(); + out << render_const_value(etype, *v_iter); + if (type->is_set()) { + out << " => true"; + } + out << "," << endl; + } + indent_down(); + indent(out) << ")"; + } + return out.str(); +} + +/** + * Make a struct + */ +void t_php_generator::generate_struct(t_struct* tstruct) { + generate_php_struct(tstruct, false); +} + +/** + * Generates a struct definition for a thrift exception. Basically the same + * as a struct but extends the Exception class. + * + * @param txception The struct definition + */ +void t_php_generator::generate_xception(t_struct* txception) { + generate_php_struct(txception, true); +} + +/** + * Structs can be normal or exceptions. + */ +void t_php_generator::generate_php_struct(t_struct* tstruct, + bool is_exception) { + generate_php_struct_definition(f_types_, tstruct, is_exception); +} + +void t_php_generator::generate_php_type_spec(ofstream& out, + t_type* t) { + t = get_true_type(t); + indent(out) << "'type' => " << type_to_enum(t) << "," << endl; + + if (t->is_base_type() || t->is_enum()) { + // Noop, type is all we need + } else if (t->is_struct() || t->is_xception()) { + indent(out) << "'class' => '" << php_namespace(t->get_program()) << t->get_name() <<"'," << endl; + } else if (t->is_map()) { + t_type* ktype = get_true_type(((t_map*)t)->get_key_type()); + t_type* vtype = get_true_type(((t_map*)t)->get_val_type()); + indent(out) << "'ktype' => " << type_to_enum(ktype) << "," << endl; + indent(out) << "'vtype' => " << type_to_enum(vtype) << "," << endl; + indent(out) << "'key' => array(" << endl; + indent_up(); + generate_php_type_spec(out, ktype); + indent_down(); + indent(out) << ")," << endl; + indent(out) << "'val' => array(" << endl; + indent_up(); + generate_php_type_spec(out, vtype); + indent(out) << ")," << endl; + indent_down(); + } else if (t->is_list() || t->is_set()) { + t_type* etype; + if (t->is_list()) { + etype = get_true_type(((t_list*)t)->get_elem_type()); + } else { + etype = get_true_type(((t_set*)t)->get_elem_type()); + } + indent(out) << "'etype' => " << type_to_enum(etype) <<"," << endl; + indent(out) << "'elem' => array(" << endl; + indent_up(); + generate_php_type_spec(out, etype); + indent(out) << ")," << endl; + indent_down(); + } else { + throw "compiler error: no type for php struct spec field"; + } + +} + +/** + * Generates the struct specification structure, which fully qualifies enough + * type information to generalize serialization routines. + */ +void t_php_generator::generate_php_struct_spec(ofstream& out, + t_struct* tstruct) { + indent(out) << "if (!isset(self::$_TSPEC)) {" << endl; + indent_up(); + + indent(out) << "self::$_TSPEC = array(" << endl; + indent_up(); + + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + t_type* t = get_true_type((*m_iter)->get_type()); + indent(out) << (*m_iter)->get_key() << " => array(" << endl; + indent_up(); + out << + indent() << "'var' => '" << (*m_iter)->get_name() << "'," << endl; + generate_php_type_spec(out, t); + indent(out) << ")," << endl; + indent_down(); + } + + indent_down(); + indent(out) << " );" << endl; + indent_down(); + indent(out) << "}" << endl; +} + + +void t_php_generator::generate_php_struct_definition(ofstream& out, + t_struct* tstruct, + bool is_exception) { + if (autoload_) { + // Make output file + ofstream autoload_out; + string f_struct = program_name_+"."+(tstruct->get_name())+".php"; + string f_struct_name = get_out_dir()+f_struct; + autoload_out.open(f_struct_name.c_str()); + autoload_out << "" << endl; + autoload_out.close(); + + f_types_ << + "$GLOBALS['THRIFT_AUTOLOAD']['" << lowercase(php_namespace(tstruct->get_program()) + tstruct->get_name()) << "'] = '" << program_name_ << "/" << f_struct << "';" << endl; + + } else { + _generate_php_struct_definition(out, tstruct, is_exception); + } +} + +/** + * Generates a struct definition for a thrift data type. This is nothing in PHP + * where the objects are all just associative arrays (unless of course we + * decide to start using objects for them...) + * + * @param tstruct The struct definition + */ +void t_php_generator::_generate_php_struct_definition(ofstream& out, + t_struct* tstruct, + bool is_exception) { + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + + out << + "class " << php_namespace(tstruct->get_program()) << tstruct->get_name(); + if (is_exception) { + out << " extends TException"; + } else if (oop_) { + out << " extends TBase"; + } + out << + " {" << endl; + indent_up(); + + indent(out) << "static $_TSPEC;" << endl << endl; + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + string dval = "null"; + t_type* t = get_true_type((*m_iter)->get_type()); + if ((*m_iter)->get_value() != NULL && !(t->is_struct() || t->is_xception())) { + dval = render_const_value((*m_iter)->get_type(), (*m_iter)->get_value()); + } + indent(out) << + "public $" << (*m_iter)->get_name() << " = " << dval << ";" << endl; + } + + out << endl; + + // Generate constructor from array + string param = (members.size() > 0) ? "$vals=null" : ""; + out << + indent() << "public function __construct(" << param << ") {" << endl; + indent_up(); + + generate_php_struct_spec(out, tstruct); + + if (members.size() > 0) { + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + t_type* t = get_true_type((*m_iter)->get_type()); + if ((*m_iter)->get_value() != NULL && (t->is_struct() || t->is_xception())) { + indent(out) << "$this->" << (*m_iter)->get_name() << " = " << render_const_value(t, (*m_iter)->get_value()) << ";" << endl; + } + } + out << + indent() << "if (is_array($vals)) {" << endl; + indent_up(); + if (oop_) { + out << indent() << "parent::__construct(self::$_TSPEC, $vals);" << endl; + } else { + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + out << + indent() << "if (isset($vals['" << (*m_iter)->get_name() << "'])) {" << endl << + indent() << " $this->" << (*m_iter)->get_name() << " = $vals['" << (*m_iter)->get_name() << "'];" << endl << + indent() << "}" << endl; + } + } + indent_down(); + out << + indent() << "}" << endl; + } + scope_down(out); + out << endl; + + out << + indent() << "public function getName() {" << endl << + indent() << " return '" << tstruct->get_name() << "';" << endl << + indent() << "}" << endl << + endl; + + generate_php_struct_reader(out, tstruct); + generate_php_struct_writer(out, tstruct); + + indent_down(); + out << + indent() << "}" << endl << + endl; +} + +/** + * Generates the read() method for a struct + */ +void t_php_generator::generate_php_struct_reader(ofstream& out, + t_struct* tstruct) { + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + indent(out) << + "public function read($input)" << endl; + scope_up(out); + + if (oop_) { + indent(out) << "return $this->_read('" << tstruct->get_name() << "', self::$_TSPEC, $input);" << endl; + scope_down(out); + return; + } + + out << + indent() << "$xfer = 0;" << endl << + indent() << "$fname = null;" << endl << + indent() << "$ftype = 0;" << endl << + indent() << "$fid = 0;" << endl; + + // Declare stack tmp variables + if (!binary_inline_) { + indent(out) << + "$xfer += $input->readStructBegin($fname);" << endl; + } + + // Loop over reading in fields + indent(out) << + "while (true)" << endl; + + scope_up(out); + + // Read beginning field marker + if (binary_inline_) { + t_field fftype(g_type_byte, "ftype"); + t_field ffid(g_type_i16, "fid"); + generate_deserialize_field(out, &fftype); + out << + indent() << "if ($ftype == TType::STOP) {" << endl << + indent() << " break;" << endl << + indent() << "}" << endl; + generate_deserialize_field(out, &ffid); + } else { + indent(out) << + "$xfer += $input->readFieldBegin($fname, $ftype, $fid);" << endl; + // Check for field STOP marker and break + indent(out) << + "if ($ftype == TType::STOP) {" << endl; + indent_up(); + indent(out) << + "break;" << endl; + indent_down(); + indent(out) << + "}" << endl; + } + + // Switch statement on the field we are reading + indent(out) << + "switch ($fid)" << endl; + + scope_up(out); + + // Generate deserialization code for known cases + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + indent(out) << + "case " << (*f_iter)->get_key() << ":" << endl; + indent_up(); + indent(out) << "if ($ftype == " << type_to_enum((*f_iter)->get_type()) << ") {" << endl; + indent_up(); + generate_deserialize_field(out, *f_iter, "this->"); + indent_down(); + out << + indent() << "} else {" << endl; + if (binary_inline_) { + indent(out) << " $xfer += TProtocol::skipBinary($input, $ftype);" << endl; + } else { + indent(out) << " $xfer += $input->skip($ftype);" << endl; + } + out << + indent() << "}" << endl << + indent() << "break;" << endl; + indent_down(); + } + + // In the default case we skip the field + indent(out) << "default:" << endl; + if (binary_inline_) { + indent(out) << " $xfer += TProtocol::skipBinary($input, $ftype);" << endl; + } else { + indent(out) << " $xfer += $input->skip($ftype);" << endl; + } + indent(out) << " break;" << endl; + + scope_down(out); + + if (!binary_inline_) { + // Read field end marker + indent(out) << + "$xfer += $input->readFieldEnd();" << endl; + } + + scope_down(out); + + if (!binary_inline_) { + indent(out) << + "$xfer += $input->readStructEnd();" << endl; + } + + indent(out) << + "return $xfer;" << endl; + + indent_down(); + out << + indent() << "}" << endl << + endl; +} + +/** + * Generates the write() method for a struct + */ +void t_php_generator::generate_php_struct_writer(ofstream& out, + t_struct* tstruct) { + string name = tstruct->get_name(); + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + if (binary_inline_) { + indent(out) << + "public function write(&$output) {" << endl; + } else { + indent(out) << + "public function write($output) {" << endl; + } + indent_up(); + + if (oop_) { + indent(out) << "return $this->_write('" << tstruct->get_name() << "', self::$_TSPEC, $output);" << endl; + scope_down(out); + return; + } + + indent(out) << + "$xfer = 0;" << endl; + + if (!binary_inline_) { + indent(out) << + "$xfer += $output->writeStructBegin('" << name << "');" << endl; + } + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + out << + indent() << "if ($this->" << (*f_iter)->get_name() << " !== null) {" << endl; + indent_up(); + + t_type* type = get_true_type((*f_iter)->get_type()); + string expect; + if (type->is_container()) { + expect = "array"; + } else if (type->is_struct()) { + expect = "object"; + } + if (!expect.empty()) { + out << + indent() << "if (!is_" << expect << "($this->" << (*f_iter)->get_name() << ")) {" << endl; + indent_up(); + out << + indent() << "throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);" << endl; + scope_down(out); + } + + // Write field header + if (binary_inline_) { + out << + indent() << "$output .= pack('c', " << type_to_enum((*f_iter)->get_type()) << ");" << endl << + indent() << "$output .= pack('n', " << (*f_iter)->get_key() << ");" << endl; + } else { + indent(out) << + "$xfer += $output->writeFieldBegin(" << + "'" << (*f_iter)->get_name() << "', " << + type_to_enum((*f_iter)->get_type()) << ", " << + (*f_iter)->get_key() << ");" << endl; + } + + // Write field contents + generate_serialize_field(out, *f_iter, "this->"); + + // Write field closer + if (!binary_inline_) { + indent(out) << + "$xfer += $output->writeFieldEnd();" << endl; + } + + indent_down(); + indent(out) << + "}" << endl; + } + + if (binary_inline_) { + out << + indent() << "$output .= pack('c', TType::STOP);" << endl; + } else { + out << + indent() << "$xfer += $output->writeFieldStop();" << endl << + indent() << "$xfer += $output->writeStructEnd();" << endl; + } + + out << + indent() << "return $xfer;" << endl; + + indent_down(); + out << + indent() << "}" << endl << + endl; +} + +/** + * Generates a thrift service. + * + * @param tservice The service definition + */ +void t_php_generator::generate_service(t_service* tservice) { + string f_service_name = get_out_dir()+service_name_+".php"; + f_service_.open(f_service_name.c_str()); + + f_service_ << + "get_extends() != NULL) { + f_service_ << + "include_once $GLOBALS['THRIFT_ROOT'].'/packages/" << tservice->get_extends()->get_program()->get_name() << "/" << tservice->get_extends()->get_name() << ".php';" << endl; + } + + f_service_ << + endl; + + // Generate the three main parts of the service (well, two for now in PHP) + generate_service_interface(tservice); + if (rest_) { + generate_service_rest(tservice); + } + generate_service_client(tservice); + generate_service_helpers(tservice); + if (phps_) { + generate_service_processor(tservice); + } + + // Close service file + f_service_ << "?>" << endl; + f_service_.close(); +} + +/** + * Generates a service server definition. + * + * @param tservice The service to generate a server for. + */ +void t_php_generator::generate_service_processor(t_service* tservice) { + // Generate the dispatch methods + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + string extends = ""; + string extends_processor = ""; + if (tservice->get_extends() != NULL) { + extends = tservice->get_extends()->get_name(); + extends_processor = " extends " + extends + "Processor"; + } + + // Generate the header portion + f_service_ << + "class " << service_name_ << "Processor" << extends_processor << " {" << endl; + indent_up(); + + if (extends.empty()) { + f_service_ << + indent() << "protected $handler_ = null;" << endl; + } + + f_service_ << + indent() << "public function __construct($handler) {" << endl; + if (extends.empty()) { + f_service_ << + indent() << " $this->handler_ = $handler;" << endl; + } else { + f_service_ << + indent() << " parent::__construct($handler);" << endl; + } + f_service_ << + indent() << "}" << endl << + endl; + + // Generate the server implementation + indent(f_service_) << + "public function process($input, $output) {" << endl; + indent_up(); + + f_service_ << + indent() << "$rseqid = 0;" << endl << + indent() << "$fname = null;" << endl << + indent() << "$mtype = 0;" << endl << + endl; + + if (binary_inline_) { + t_field ffname(g_type_string, "fname"); + t_field fmtype(g_type_byte, "mtype"); + t_field fseqid(g_type_i32, "rseqid"); + generate_deserialize_field(f_service_, &ffname, "", true); + generate_deserialize_field(f_service_, &fmtype, "", true); + generate_deserialize_field(f_service_, &fseqid, "", true); + } else { + f_service_ << + indent() << "$input->readMessageBegin($fname, $mtype, $rseqid);" << endl; + } + + // HOT: check for method implementation + f_service_ << + indent() << "$methodname = 'process_'.$fname;" << endl << + indent() << "if (!method_exists($this, $methodname)) {" << endl; + if (binary_inline_) { + f_service_ << + indent() << " throw new Exception('Function '.$fname.' not implemented.');" << endl; + } else { + f_service_ << + indent() << " $input->skip(TType::STRUCT);" << endl << + indent() << " $input->readMessageEnd();" << endl << + indent() << " $x = new TApplicationException('Function '.$fname.' not implemented.', TApplicationException::UNKNOWN_METHOD);" << endl << + indent() << " $output->writeMessageBegin($fname, TMessageType::EXCEPTION, $rseqid);" << endl << + indent() << " $x->write($output);" << endl << + indent() << " $output->writeMessageEnd();" << endl << + indent() << " $output->getTransport()->flush();" << endl << + indent() << " return;" << endl; + } + f_service_ << + indent() << "}" << endl << + indent() << "$this->$methodname($rseqid, $input, $output);" << endl << + indent() << "return true;" << endl; + indent_down(); + f_service_ << + indent() << "}" << endl << + endl; + + // Generate the process subfunctions + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + generate_process_function(tservice, *f_iter); + } + + indent_down(); + f_service_ << "}" << endl; +} + +/** + * Generates a process function definition. + * + * @param tfunction The function to write a dispatcher for + */ +void t_php_generator::generate_process_function(t_service* tservice, + t_function* tfunction) { + // Open function + indent(f_service_) << + "protected function process_" << tfunction->get_name() << + "($seqid, $input, $output) {" << endl; + indent_up(); + + string argsname = php_namespace(tservice->get_program()) + service_name_ + "_" + tfunction->get_name() + "_args"; + string resultname = php_namespace(tservice->get_program()) + service_name_ + "_" + tfunction->get_name() + "_result"; + + f_service_ << + indent() << "$args = new " << argsname << "();" << endl << + indent() << "$args->read($input);" << endl; + if (!binary_inline_) { + f_service_ << + indent() << "$input->readMessageEnd();" << endl; + } + + t_struct* xs = tfunction->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + + // Declare result for non oneway function + if (!tfunction->is_oneway()) { + f_service_ << + indent() << "$result = new " << resultname << "();" << endl; + } + + // Try block for a function with exceptions + if (xceptions.size() > 0) { + f_service_ << + indent() << "try {" << endl; + indent_up(); + } + + // Generate the function call + t_struct* arg_struct = tfunction->get_arglist(); + const std::vector& fields = arg_struct->get_members(); + vector::const_iterator f_iter; + + f_service_ << indent(); + if (!tfunction->is_oneway() && !tfunction->get_returntype()->is_void()) { + f_service_ << "$result->success = "; + } + f_service_ << + "$this->handler_->" << tfunction->get_name() << "("; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << "$args->" << (*f_iter)->get_name(); + } + f_service_ << ");" << endl; + + if (!tfunction->is_oneway() && xceptions.size() > 0) { + indent_down(); + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << + indent() << "} catch (" << php_namespace((*x_iter)->get_type()->get_program()) << (*x_iter)->get_type()->get_name() << " $" << (*x_iter)->get_name() << ") {" << endl; + if (!tfunction->is_oneway()) { + indent_up(); + f_service_ << + indent() << "$result->" << (*x_iter)->get_name() << " = $" << (*x_iter)->get_name() << ";" << endl; + indent_down(); + f_service_ << indent(); + } + } + f_service_ << "}" << endl; + } + + // Shortcut out here for oneway functions + if (tfunction->is_oneway()) { + f_service_ << + indent() << "return;" << endl; + indent_down(); + f_service_ << + indent() << "}" << endl; + return; + } + + // Serialize the request header + if (binary_inline_) { + f_service_ << + indent() << "$buff = pack('N', (0x80010000 | TMessageType::REPLY)); " << endl << + indent() << "$buff .= pack('N', strlen('" << tfunction->get_name() << "'));" << endl << + indent() << "$buff .= '" << tfunction->get_name() << "';" << endl << + indent() << "$buff .= pack('N', $seqid);" << endl << + indent() << "$result->write($buff);" << endl << + indent() << "$output->write($buff);" << endl << + indent() << "$output->flush();" << endl; + } else { + f_service_ << + indent() << "$output->writeMessageBegin('" << tfunction->get_name() << "', TMessageType::REPLY, $seqid);" << endl << + indent() << "$result->write($output);" << endl << + indent() << "$output->getTransport()->flush();" << endl; + } + + // Close function + indent_down(); + f_service_ << + indent() << "}" << endl; +} + +/** + * Generates helper functions for a service. + * + * @param tservice The service to generate a header definition for + */ +void t_php_generator::generate_service_helpers(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + f_service_ << + "// HELPER FUNCTIONS AND STRUCTURES" << endl << endl; + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* ts = (*f_iter)->get_arglist(); + string name = ts->get_name(); + ts->set_name(service_name_ + "_" + name); + generate_php_struct_definition(f_service_, ts, false); + generate_php_function_helpers(*f_iter); + ts->set_name(name); + } +} + +/** + * Generates a struct and helpers for a function. + * + * @param tfunction The function + */ +void t_php_generator::generate_php_function_helpers(t_function* tfunction) { + if (!tfunction->is_oneway()) { + t_struct result(program_, service_name_ + "_" + tfunction->get_name() + "_result"); + t_field success(tfunction->get_returntype(), "success", 0); + if (!tfunction->get_returntype()->is_void()) { + result.append(&success); + } + + t_struct* xs = tfunction->get_xceptions(); + const vector& fields = xs->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + result.append(*f_iter); + } + + generate_php_struct_definition(f_service_, &result, false); + } +} + +/** + * Generates a service interface definition. + * + * @param tservice The service to generate a header definition for + */ +void t_php_generator::generate_service_interface(t_service* tservice) { + string extends = ""; + string extends_if = ""; + if (tservice->get_extends() != NULL) { + extends = " extends " + tservice->get_extends()->get_name(); + extends_if = " extends " + tservice->get_extends()->get_name() + "If"; + } + f_service_ << + "interface " << service_name_ << "If" << extends_if << " {" << endl; + indent_up(); + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + indent(f_service_) << + "public function " << function_signature(*f_iter) << ";" << endl; + } + indent_down(); + f_service_ << + "}" << endl << endl; +} + +/** + * Generates a REST interface + */ +void t_php_generator::generate_service_rest(t_service* tservice) { + string extends = ""; + string extends_if = ""; + if (tservice->get_extends() != NULL) { + extends = " extends " + tservice->get_extends()->get_name(); + extends_if = " extends " + tservice->get_extends()->get_name() + "Rest"; + } + f_service_ << + "class " << service_name_ << "Rest" << extends_if << " {" << endl; + indent_up(); + + if (extends.empty()) { + f_service_ << + indent() << "protected $impl_;" << endl << + endl; + } + + f_service_ << + indent() << "public function __construct($impl) {" << endl << + indent() << " $this->impl_ = $impl;" << endl << + indent() << "}" << endl << + endl; + + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + indent(f_service_) << + "public function " << (*f_iter)->get_name() << "($request) {" << endl; + indent_up(); + const vector& args = (*f_iter)->get_arglist()->get_members(); + vector::const_iterator a_iter; + for (a_iter = args.begin(); a_iter != args.end(); ++a_iter) { + t_type* atype = get_true_type((*a_iter)->get_type()); + string cast = type_to_cast(atype); + string req = "$request['" + (*a_iter)->get_name() + "']"; + if (atype->is_bool()) { + f_service_ << + indent() << "$" << (*a_iter)->get_name() << " = " << cast << "(!empty(" << req << ") && (" << req << " !== 'false'));" << endl; + } else { + f_service_ << + indent() << "$" << (*a_iter)->get_name() << " = isset(" << req << ") ? " << cast << req << " : null;" << endl; + } + if (atype->is_string() && + ((t_base_type*)atype)->is_string_list()) { + f_service_ << + indent() << "$" << (*a_iter)->get_name() << " = explode(',', $" << (*a_iter)->get_name() << ");" << endl; + } else if (atype->is_map() || atype->is_list()) { + f_service_ << + indent() << "$" << (*a_iter)->get_name() << " = json_decode($" << (*a_iter)->get_name() << ", true);" << endl; + } else if (atype->is_set()) { + f_service_ << + indent() << "$" << (*a_iter)->get_name() << " = array_fill_keys(json_decode($" << (*a_iter)->get_name() << ", true), 1);" << endl; + } else if (atype->is_struct() || atype->is_xception()) { + f_service_ << + indent() << "if ($" << (*a_iter)->get_name() << " !== null) {" << endl << + indent() << " $" << (*a_iter)->get_name() << " = new " << php_namespace(atype->get_program()) << atype->get_name() << "(json_decode($" << (*a_iter)->get_name() << ", true));" << endl << + indent() << "}" << endl; + } + } + f_service_ << + indent() << "return $this->impl_->" << (*f_iter)->get_name() << "(" << argument_list((*f_iter)->get_arglist()) << ");" << endl; + indent_down(); + indent(f_service_) << + "}" << endl << + endl; + } + indent_down(); + f_service_ << + "}" << endl << endl; +} + +void t_php_generator::generate_service_client(t_service* tservice) { + if (autoload_) { + // Make output file + ofstream autoload_out; + string f_struct = program_name_+"."+(tservice->get_name())+".client.php"; + string f_struct_name = get_out_dir()+f_struct; + autoload_out.open(f_struct_name.c_str()); + autoload_out << "" << endl; + autoload_out.close(); + + f_service_ << + "$GLOBALS['THRIFT_AUTOLOAD']['" << lowercase(service_name_ + "Client") << "'] = '" << program_name_ << "/" << f_struct << "';" << endl; + + } else { + _generate_service_client(f_service_, tservice); + } +} + +/** + * Generates a service client definition. + * + * @param tservice The service to generate a server for. + */ +void t_php_generator::_generate_service_client(ofstream& out, t_service* tservice) { + string extends = ""; + string extends_client = ""; + if (tservice->get_extends() != NULL) { + extends = tservice->get_extends()->get_name(); + extends_client = " extends " + extends + "Client"; + } + + out << + "class " << service_name_ << "Client" << extends_client << " implements " << service_name_ << "If {" << endl; + indent_up(); + + // Private members + if (extends.empty()) { + out << + indent() << "protected $input_ = null;" << endl << + indent() << "protected $output_ = null;" << endl << + endl; + out << + indent() << "protected $seqid_ = 0;" << endl << + endl; + } + + // Constructor function + out << + indent() << "public function __construct($input, $output=null) {" << endl; + if (!extends.empty()) { + out << + indent() << " parent::__construct($input, $output);" << endl; + } else { + out << + indent() << " $this->input_ = $input;" << endl << + indent() << " $this->output_ = $output ? $output : $input;" << endl; + } + out << + indent() << "}" << endl << endl; + + // Generate client method implementations + vector functions = tservice->get_functions(); + vector::const_iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* arg_struct = (*f_iter)->get_arglist(); + const vector& fields = arg_struct->get_members(); + vector::const_iterator fld_iter; + string funname = (*f_iter)->get_name(); + + // Open function + indent(out) << + "public function " << function_signature(*f_iter) << endl; + scope_up(out); + indent(out) << + "$this->send_" << funname << "("; + + bool first = true; + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + if (first) { + first = false; + } else { + out << ", "; + } + out << "$" << (*fld_iter)->get_name(); + } + out << ");" << endl; + + if (!(*f_iter)->is_oneway()) { + out << indent(); + if (!(*f_iter)->get_returntype()->is_void()) { + out << "return "; + } + out << + "$this->recv_" << funname << "();" << endl; + } + scope_down(out); + out << endl; + + indent(out) << + "public function send_" << function_signature(*f_iter) << endl; + scope_up(out); + + std::string argsname = php_namespace(tservice->get_program()) + service_name_ + "_" + (*f_iter)->get_name() + "_args"; + + out << + indent() << "$args = new " << argsname << "();" << endl; + + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + out << + indent() << "$args->" << (*fld_iter)->get_name() << " = $" << (*fld_iter)->get_name() << ";" << endl; + } + + out << + indent() << "$bin_accel = ($this->output_ instanceof TProtocol::$TBINARYPROTOCOLACCELERATED) && function_exists('thrift_protocol_write_binary');" << endl; + + out << + indent() << "if ($bin_accel)" << endl; + scope_up(out); + + out << + indent() << "thrift_protocol_write_binary($this->output_, '" << (*f_iter)->get_name() << "', TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite());" << endl; + + scope_down(out); + out << + indent() << "else" << endl; + scope_up(out); + + // Serialize the request header + if (binary_inline_) { + out << + indent() << "$buff = pack('N', (0x80010000 | TMessageType::CALL));" << endl << + indent() << "$buff .= pack('N', strlen('" << funname << "'));" << endl << + indent() << "$buff .= '" << funname << "';" << endl << + indent() << "$buff .= pack('N', $this->seqid_);" << endl; + } else { + out << + indent() << "$this->output_->writeMessageBegin('" << (*f_iter)->get_name() << "', TMessageType::CALL, $this->seqid_);" << endl; + } + + // Write to the stream + if (binary_inline_) { + out << + indent() << "$args->write($buff);" << endl << + indent() << "$this->output_->write($buff);" << endl << + indent() << "$this->output_->flush();" << endl; + } else { + out << + indent() << "$args->write($this->output_);" << endl << + indent() << "$this->output_->writeMessageEnd();" << endl << + indent() << "$this->output_->getTransport()->flush();" << endl; + } + + scope_down(out); + + scope_down(out); + + + if (!(*f_iter)->is_oneway()) { + std::string resultname = php_namespace(tservice->get_program()) + service_name_ + "_" + (*f_iter)->get_name() + "_result"; + t_struct noargs(program_); + + t_function recv_function((*f_iter)->get_returntype(), + string("recv_") + (*f_iter)->get_name(), + &noargs); + // Open function + out << + endl << + indent() << "public function " << function_signature(&recv_function) << endl; + scope_up(out); + + out << + indent() << "$bin_accel = ($this->input_ instanceof TProtocol::$TBINARYPROTOCOLACCELERATED)" + << " && function_exists('thrift_protocol_read_binary');" << endl; + + out << + indent() << "if ($bin_accel) $result = thrift_protocol_read_binary($this->input_, '" << resultname << "', $this->input_->isStrictRead());" << endl; + out << + indent() << "else" << endl; + scope_up(out); + + out << + indent() << "$rseqid = 0;" << endl << + indent() << "$fname = null;" << endl << + indent() << "$mtype = 0;" << endl << + endl; + + if (binary_inline_) { + t_field ffname(g_type_string, "fname"); + t_field fseqid(g_type_i32, "rseqid"); + out << + indent() << "$ver = unpack('N', $this->input_->readAll(4));" << endl << + indent() << "$ver = $ver[1];" << endl << + indent() << "$mtype = $ver & 0xff;" << endl << + indent() << "$ver = $ver & 0xffff0000;" << endl << + indent() << "if ($ver != 0x80010000) throw new TProtocolException('Bad version identifier: '.$ver, TProtocolException::BAD_VERSION);" << endl; + generate_deserialize_field(out, &ffname, "", true); + generate_deserialize_field(out, &fseqid, "", true); + } else { + out << + indent() << "$this->input_->readMessageBegin($fname, $mtype, $rseqid);" << endl << + indent() << "if ($mtype == TMessageType::EXCEPTION) {" << endl << + indent() << " $x = new TApplicationException();" << endl << + indent() << " $x->read($this->input_);" << endl << + indent() << " $this->input_->readMessageEnd();" << endl << + indent() << " throw $x;" << endl << + indent() << "}" << endl; + } + + out << + indent() << "$result = new " << resultname << "();" << endl << + indent() << "$result->read($this->input_);" << endl; + + if (!binary_inline_) { + out << + indent() << "$this->input_->readMessageEnd();" << endl; + } + + scope_down(out); + + // Careful, only return result if not a void function + if (!(*f_iter)->get_returntype()->is_void()) { + out << + indent() << "if ($result->success !== null) {" << endl << + indent() << " return $result->success;" << endl << + indent() << "}" << endl; + } + + t_struct* xs = (*f_iter)->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + out << + indent() << "if ($result->" << (*x_iter)->get_name() << " !== null) {" << endl << + indent() << " throw $result->" << (*x_iter)->get_name() << ";" << endl << + indent() << "}" << endl; + } + + // Careful, only return _result if not a void function + if ((*f_iter)->get_returntype()->is_void()) { + indent(out) << + "return;" << endl; + } else { + out << + indent() << "throw new Exception(\"" << (*f_iter)->get_name() << " failed: unknown result\");" << endl; + } + + // Close function + scope_down(out); + out << endl; + + } + } + + indent_down(); + out << + "}" << endl << endl; +} + +/** + * Deserializes a field of any type. + */ +void t_php_generator::generate_deserialize_field(ofstream &out, + t_field* tfield, + string prefix, + bool inclass) { + t_type* type = get_true_type(tfield->get_type()); + + if (type->is_void()) { + throw "CANNOT GENERATE DESERIALIZE CODE FOR void TYPE: " + + prefix + tfield->get_name(); + } + + string name = prefix + tfield->get_name(); + + if (type->is_struct() || type->is_xception()) { + generate_deserialize_struct(out, + (t_struct*)type, + name); + } else { + + if (type->is_container()) { + generate_deserialize_container(out, type, name); + } else if (type->is_base_type() || type->is_enum()) { + + if (binary_inline_) { + std::string itrans = (inclass ? "$this->input_" : "$input"); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "compiler error: cannot serialize void field in a struct: " + + name; + break; + case t_base_type::TYPE_STRING: + out << + indent() << "$len = unpack('N', " << itrans << "->readAll(4));" << endl << + indent() << "$len = $len[1];" << endl << + indent() << "if ($len > 0x7fffffff) {" << endl << + indent() << " $len = 0 - (($len - 1) ^ 0xffffffff);" << endl << + indent() << "}" << endl << + indent() << "$" << name << " = " << itrans << "->readAll($len);" << endl; + break; + case t_base_type::TYPE_BOOL: + out << + indent() << "$" << name << " = unpack('c', " << itrans << "->readAll(1));" << endl << + indent() << "$" << name << " = (bool)$" << name << "[1];" << endl; + break; + case t_base_type::TYPE_BYTE: + out << + indent() << "$" << name << " = unpack('c', " << itrans << "->readAll(1));" << endl << + indent() << "$" << name << " = $" << name << "[1];" << endl; + break; + case t_base_type::TYPE_I16: + out << + indent() << "$val = unpack('n', " << itrans << "->readAll(2));" << endl << + indent() << "$val = $val[1];" << endl << + indent() << "if ($val > 0x7fff) {" << endl << + indent() << " $val = 0 - (($val - 1) ^ 0xffff);" << endl << + indent() << "}" << endl << + indent() << "$" << name << " = $val;" << endl; + break; + case t_base_type::TYPE_I32: + out << + indent() << "$val = unpack('N', " << itrans << "->readAll(4));" << endl << + indent() << "$val = $val[1];" << endl << + indent() << "if ($val > 0x7fffffff) {" << endl << + indent() << " $val = 0 - (($val - 1) ^ 0xffffffff);" << endl << + indent() << "}" << endl << + indent() << "$" << name << " = $val;" << endl; + break; + case t_base_type::TYPE_I64: + out << + indent() << "$arr = unpack('N2', " << itrans << "->readAll(8));" << endl << + indent() << "if ($arr[1] & 0x80000000) {" << endl << + indent() << " $arr[1] = $arr[1] ^ 0xFFFFFFFF;" << endl << + indent() << " $arr[2] = $arr[2] ^ 0xFFFFFFFF;" << endl << + indent() << " $" << name << " = 0 - $arr[1]*4294967296 - $arr[2] - 1;" << endl << + indent() << "} else {" << endl << + indent() << " $" << name << " = $arr[1]*4294967296 + $arr[2];" << endl << + indent() << "}" << endl; + break; + case t_base_type::TYPE_DOUBLE: + out << + indent() << "$arr = unpack('d', strrev(" << itrans << "->readAll(8)));" << endl << + indent() << "$" << name << " = $arr[1];" << endl; + break; + default: + throw "compiler error: no PHP name for base type " + t_base_type::t_base_name(tbase) + tfield->get_name(); + } + } else if (type->is_enum()) { + out << + indent() << "$val = unpack('N', " << itrans << "->readAll(4));" << endl << + indent() << "$val = $val[1];" << endl << + indent() << "if ($val > 0x7fffffff) {" << endl << + indent() << " $val = 0 - (($val - 1) ^ 0xffffffff);" << endl << + indent() << "}" << endl << + indent() << "$" << name << " = $val;" << endl; + } + } else { + + indent(out) << + "$xfer += $input->"; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "compiler error: cannot serialize void field in a struct: " + + name; + break; + case t_base_type::TYPE_STRING: + out << "readString($" << name << ");"; + break; + case t_base_type::TYPE_BOOL: + out << "readBool($" << name << ");"; + break; + case t_base_type::TYPE_BYTE: + out << "readByte($" << name << ");"; + break; + case t_base_type::TYPE_I16: + out << "readI16($" << name << ");"; + break; + case t_base_type::TYPE_I32: + out << "readI32($" << name << ");"; + break; + case t_base_type::TYPE_I64: + out << "readI64($" << name << ");"; + break; + case t_base_type::TYPE_DOUBLE: + out << "readDouble($" << name << ");"; + break; + default: + throw "compiler error: no PHP name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + out << "readI32($" << name << ");"; + } + out << endl; + } + } else { + printf("DO NOT KNOW HOW TO DESERIALIZE FIELD '%s' TYPE '%s'\n", + tfield->get_name().c_str(), type->get_name().c_str()); + } + } +} + +/** + * Generates an unserializer for a variable. This makes two key assumptions, + * first that there is a const char* variable named data that points to the + * buffer for deserialization, and that there is a variable protocol which + * is a reference to a TProtocol serialization object. + */ +void t_php_generator::generate_deserialize_struct(ofstream &out, + t_struct* tstruct, + string prefix) { + out << + indent() << "$" << prefix << " = new " << php_namespace(tstruct->get_program()) << tstruct->get_name() << "();" << endl << + indent() << "$xfer += $" << prefix << "->read($input);" << endl; +} + +void t_php_generator::generate_deserialize_container(ofstream &out, + t_type* ttype, + string prefix) { + string size = tmp("_size"); + string ktype = tmp("_ktype"); + string vtype = tmp("_vtype"); + string etype = tmp("_etype"); + + t_field fsize(g_type_i32, size); + t_field fktype(g_type_byte, ktype); + t_field fvtype(g_type_byte, vtype); + t_field fetype(g_type_byte, etype); + + out << + indent() << "$" << prefix << " = array();" << endl << + indent() << "$" << size << " = 0;" << endl; + + // Declare variables, read header + if (ttype->is_map()) { + out << + indent() << "$" << ktype << " = 0;" << endl << + indent() << "$" << vtype << " = 0;" << endl; + if (binary_inline_) { + generate_deserialize_field(out, &fktype); + generate_deserialize_field(out, &fvtype); + generate_deserialize_field(out, &fsize); + } else { + out << + indent() << "$xfer += $input->readMapBegin(" << + "$" << ktype << ", $" << vtype << ", $" << size << ");" << endl; + } + } else if (ttype->is_set()) { + if (binary_inline_) { + generate_deserialize_field(out, &fetype); + generate_deserialize_field(out, &fsize); + } else { + out << + indent() << "$" << etype << " = 0;" << endl << + indent() << "$xfer += $input->readSetBegin(" << + "$" << etype << ", $" << size << ");" << endl; + } + } else if (ttype->is_list()) { + if (binary_inline_) { + generate_deserialize_field(out, &fetype); + generate_deserialize_field(out, &fsize); + } else { + out << + indent() << "$" << etype << " = 0;" << endl << + indent() << "$xfer += $input->readListBegin(" << + "$" << etype << ", $" << size << ");" << endl; + } + } + + // For loop iterates over elements + string i = tmp("_i"); + indent(out) << + "for ($" << + i << " = 0; $" << i << " < $" << size << "; ++$" << i << ")" << endl; + + scope_up(out); + + if (ttype->is_map()) { + generate_deserialize_map_element(out, (t_map*)ttype, prefix); + } else if (ttype->is_set()) { + generate_deserialize_set_element(out, (t_set*)ttype, prefix); + } else if (ttype->is_list()) { + generate_deserialize_list_element(out, (t_list*)ttype, prefix); + } + + scope_down(out); + + if (!binary_inline_) { + // Read container end + if (ttype->is_map()) { + indent(out) << "$xfer += $input->readMapEnd();" << endl; + } else if (ttype->is_set()) { + indent(out) << "$xfer += $input->readSetEnd();" << endl; + } else if (ttype->is_list()) { + indent(out) << "$xfer += $input->readListEnd();" << endl; + } + } +} + + +/** + * Generates code to deserialize a map + */ +void t_php_generator::generate_deserialize_map_element(ofstream &out, + t_map* tmap, + string prefix) { + string key = tmp("key"); + string val = tmp("val"); + t_field fkey(tmap->get_key_type(), key); + t_field fval(tmap->get_val_type(), val); + + indent(out) << + declare_field(&fkey, true, true) << endl; + indent(out) << + declare_field(&fval, true, true) << endl; + + generate_deserialize_field(out, &fkey); + generate_deserialize_field(out, &fval); + + indent(out) << + "$" << prefix << "[$" << key << "] = $" << val << ";" << endl; +} + +void t_php_generator::generate_deserialize_set_element(ofstream &out, + t_set* tset, + string prefix) { + string elem = tmp("elem"); + t_field felem(tset->get_elem_type(), elem); + + indent(out) << + "$" << elem << " = null;" << endl; + + generate_deserialize_field(out, &felem); + + indent(out) << + "$" << prefix << "[$" << elem << "] = true;" << endl; +} + +void t_php_generator::generate_deserialize_list_element(ofstream &out, + t_list* tlist, + string prefix) { + string elem = tmp("elem"); + t_field felem(tlist->get_elem_type(), elem); + + indent(out) << + "$" << elem << " = null;" << endl; + + generate_deserialize_field(out, &felem); + + indent(out) << + "$" << prefix << " []= $" << elem << ";" << endl; +} + + +/** + * Serializes a field of any type. + * + * @param tfield The field to serialize + * @param prefix Name to prepend to field name + */ +void t_php_generator::generate_serialize_field(ofstream &out, + t_field* tfield, + string prefix) { + t_type* type = get_true_type(tfield->get_type()); + + // Do nothing for void types + if (type->is_void()) { + throw "CANNOT GENERATE SERIALIZE CODE FOR void TYPE: " + + prefix + tfield->get_name(); + } + + if (type->is_struct() || type->is_xception()) { + generate_serialize_struct(out, + (t_struct*)type, + prefix + tfield->get_name()); + } else if (type->is_container()) { + generate_serialize_container(out, + type, + prefix + tfield->get_name()); + } else if (type->is_base_type() || type->is_enum()) { + + string name = prefix + tfield->get_name(); + + if (binary_inline_) { + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw + "compiler error: cannot serialize void field in a struct: " + name; + break; + case t_base_type::TYPE_STRING: + out << + indent() << "$output .= pack('N', strlen($" << name << "));" << endl << + indent() << "$output .= $" << name << ";" << endl; + break; + case t_base_type::TYPE_BOOL: + out << + indent() << "$output .= pack('c', $" << name << " ? 1 : 0);" << endl; + break; + case t_base_type::TYPE_BYTE: + out << + indent() << "$output .= pack('c', $" << name << ");" << endl; + break; + case t_base_type::TYPE_I16: + out << + indent() << "$output .= pack('n', $" << name << ");" << endl; + break; + case t_base_type::TYPE_I32: + out << + indent() << "$output .= pack('N', $" << name << ");" << endl; + break; + case t_base_type::TYPE_I64: + out << + indent() << "$output .= pack('N2', $" << name << " >> 32, $" << name << " & 0xFFFFFFFF);" << endl; + break; + case t_base_type::TYPE_DOUBLE: + out << + indent() << "$output .= strrev(pack('d', $" << name << "));" << endl; + break; + default: + throw "compiler error: no PHP name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + out << + indent() << "$output .= pack('N', $" << name << ");" << endl; + } + } else { + + indent(out) << + "$xfer += $output->"; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw + "compiler error: cannot serialize void field in a struct: " + name; + break; + case t_base_type::TYPE_STRING: + out << "writeString($" << name << ");"; + break; + case t_base_type::TYPE_BOOL: + out << "writeBool($" << name << ");"; + break; + case t_base_type::TYPE_BYTE: + out << "writeByte($" << name << ");"; + break; + case t_base_type::TYPE_I16: + out << "writeI16($" << name << ");"; + break; + case t_base_type::TYPE_I32: + out << "writeI32($" << name << ");"; + break; + case t_base_type::TYPE_I64: + out << "writeI64($" << name << ");"; + break; + case t_base_type::TYPE_DOUBLE: + out << "writeDouble($" << name << ");"; + break; + default: + throw "compiler error: no PHP name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + out << "writeI32($" << name << ");"; + } + out << endl; + } + } else { + printf("DO NOT KNOW HOW TO SERIALIZE FIELD '%s%s' TYPE '%s'\n", + prefix.c_str(), + tfield->get_name().c_str(), + type->get_name().c_str()); + } +} + +/** + * Serializes all the members of a struct. + * + * @param tstruct The struct to serialize + * @param prefix String prefix to attach to all fields + */ +void t_php_generator::generate_serialize_struct(ofstream &out, + t_struct* tstruct, + string prefix) { + indent(out) << + "$xfer += $" << prefix << "->write($output);" << endl; +} + +/** + * Writes out a container + */ +void t_php_generator::generate_serialize_container(ofstream &out, + t_type* ttype, + string prefix) { + scope_up(out); + + if (ttype->is_map()) { + if (binary_inline_) { + out << + indent() << "$output .= pack('c', " << type_to_enum(((t_map*)ttype)->get_key_type()) << ");" << endl << + indent() << "$output .= pack('c', " << type_to_enum(((t_map*)ttype)->get_val_type()) << ");" << endl << + indent() << "$output .= strrev(pack('l', count($" << prefix << ")));" << endl; + } else { + indent(out) << + "$output->writeMapBegin(" << + type_to_enum(((t_map*)ttype)->get_key_type()) << ", " << + type_to_enum(((t_map*)ttype)->get_val_type()) << ", " << + "count($" << prefix << "));" << endl; + } + } else if (ttype->is_set()) { + if (binary_inline_) { + out << + indent() << "$output .= pack('c', " << type_to_enum(((t_set*)ttype)->get_elem_type()) << ");" << endl << + indent() << "$output .= strrev(pack('l', count($" << prefix << ")));" << endl; + + } else { + indent(out) << + "$output->writeSetBegin(" << + type_to_enum(((t_set*)ttype)->get_elem_type()) << ", " << + "count($" << prefix << "));" << endl; + } + } else if (ttype->is_list()) { + if (binary_inline_) { + out << + indent() << "$output .= pack('c', " << type_to_enum(((t_list*)ttype)->get_elem_type()) << ");" << endl << + indent() << "$output .= strrev(pack('l', count($" << prefix << ")));" << endl; + + } else { + indent(out) << + "$output->writeListBegin(" << + type_to_enum(((t_list*)ttype)->get_elem_type()) << ", " << + "count($" << prefix << "));" << endl; + } + } + + scope_up(out); + + if (ttype->is_map()) { + string kiter = tmp("kiter"); + string viter = tmp("viter"); + indent(out) << + "foreach ($" << prefix << " as " << + "$" << kiter << " => $" << viter << ")" << endl; + scope_up(out); + generate_serialize_map_element(out, (t_map*)ttype, kiter, viter); + scope_down(out); + } else if (ttype->is_set()) { + string iter = tmp("iter"); + indent(out) << + "foreach ($" << prefix << " as $" << iter << " => $true)" << endl; + scope_up(out); + generate_serialize_set_element(out, (t_set*)ttype, iter); + scope_down(out); + } else if (ttype->is_list()) { + string iter = tmp("iter"); + indent(out) << + "foreach ($" << prefix << " as $" << iter << ")" << endl; + scope_up(out); + generate_serialize_list_element(out, (t_list*)ttype, iter); + scope_down(out); + } + + scope_down(out); + + if (!binary_inline_) { + if (ttype->is_map()) { + indent(out) << + "$output->writeMapEnd();" << endl; + } else if (ttype->is_set()) { + indent(out) << + "$output->writeSetEnd();" << endl; + } else if (ttype->is_list()) { + indent(out) << + "$output->writeListEnd();" << endl; + } + } + + scope_down(out); +} + +/** + * Serializes the members of a map. + * + */ +void t_php_generator::generate_serialize_map_element(ofstream &out, + t_map* tmap, + string kiter, + string viter) { + t_field kfield(tmap->get_key_type(), kiter); + generate_serialize_field(out, &kfield, ""); + + t_field vfield(tmap->get_val_type(), viter); + generate_serialize_field(out, &vfield, ""); +} + +/** + * Serializes the members of a set. + */ +void t_php_generator::generate_serialize_set_element(ofstream &out, + t_set* tset, + string iter) { + t_field efield(tset->get_elem_type(), iter); + generate_serialize_field(out, &efield, ""); +} + +/** + * Serializes the members of a list. + */ +void t_php_generator::generate_serialize_list_element(ofstream &out, + t_list* tlist, + string iter) { + t_field efield(tlist->get_elem_type(), iter); + generate_serialize_field(out, &efield, ""); +} + +/** + * Declares a field, which may include initialization as necessary. + * + * @param ttype The type + */ +string t_php_generator::declare_field(t_field* tfield, bool init, bool obj) { + string result = "$" + tfield->get_name(); + if (init) { + t_type* type = get_true_type(tfield->get_type()); + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + break; + case t_base_type::TYPE_STRING: + result += " = ''"; + break; + case t_base_type::TYPE_BOOL: + result += " = false"; + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + result += " = 0"; + break; + case t_base_type::TYPE_DOUBLE: + result += " = 0.0"; + break; + default: + throw "compiler error: no PHP initializer for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + result += " = 0"; + } else if (type->is_container()) { + result += " = array()"; + } else if (type->is_struct() || type->is_xception()) { + if (obj) { + result += " = new " + php_namespace(type->get_program()) + type->get_name() + "()"; + } else { + result += " = null"; + } + } + } + return result + ";"; +} + +/** + * Renders a function signature of the form 'type name(args)' + * + * @param tfunction Function definition + * @return String of rendered function definition + */ +string t_php_generator::function_signature(t_function* tfunction, + string prefix) { + return + prefix + tfunction->get_name() + + "(" + argument_list(tfunction->get_arglist()) + ")"; +} + +/** + * Renders a field list + */ +string t_php_generator::argument_list(t_struct* tstruct) { + string result = ""; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + result += ", "; + } + result += "$" + (*f_iter)->get_name(); + } + return result; +} + +/** + * Gets a typecast string for a particular type. + */ +string t_php_generator::type_to_cast(t_type* type) { + if (type->is_base_type()) { + t_base_type* btype = (t_base_type*)type; + switch (btype->get_base()) { + case t_base_type::TYPE_BOOL: + return "(bool)"; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + return "(int)"; + case t_base_type::TYPE_DOUBLE: + return "(double)"; + case t_base_type::TYPE_STRING: + return "(string)"; + default: + return ""; + } + } else if (type->is_enum()) { + return "(int)"; + } + return ""; +} + +/** + * Converts the parse type to a C++ enum string for the given type. + */ +string t_php_generator ::type_to_enum(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + return "TType::STRING"; + case t_base_type::TYPE_BOOL: + return "TType::BOOL"; + case t_base_type::TYPE_BYTE: + return "TType::BYTE"; + case t_base_type::TYPE_I16: + return "TType::I16"; + case t_base_type::TYPE_I32: + return "TType::I32"; + case t_base_type::TYPE_I64: + return "TType::I64"; + case t_base_type::TYPE_DOUBLE: + return "TType::DOUBLE"; + } + } else if (type->is_enum()) { + return "TType::I32"; + } else if (type->is_struct() || type->is_xception()) { + return "TType::STRUCT"; + } else if (type->is_map()) { + return "TType::MAP"; + } else if (type->is_set()) { + return "TType::SET"; + } else if (type->is_list()) { + return "TType::LST"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + +THRIFT_REGISTER_GENERATOR(php, "PHP", +" inlined: Generate PHP inlined files\n" +" server: Generate PHP server stubs\n" +" autoload: Generate PHP with autoload\n" +" oop: Generate PHP with object oriented subclasses\n" +" rest: Generate PHP REST processors\n" +); diff --git a/compiler/cpp/src/generate/t_py_generator.cc b/compiler/cpp/src/generate/t_py_generator.cc new file mode 100644 index 00000000..343c982b --- /dev/null +++ b/compiler/cpp/src/generate/t_py_generator.cc @@ -0,0 +1,2310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include "t_generator.h" +#include "platform.h" +using namespace std; + + +/** + * Python code generator. + * + */ +class t_py_generator : public t_generator { + public: + t_py_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_generator(program) + { + std::map::const_iterator iter; + + iter = parsed_options.find("new_style"); + gen_newstyle_ = (iter != parsed_options.end()); + + iter = parsed_options.find("twisted"); + gen_twisted_ = (iter != parsed_options.end()); + + out_dir_base_ = "gen-py"; + } + + /** + * Init and close methods + */ + + void init_generator(); + void close_generator(); + + /** + * Program-level generation functions + */ + + void generate_typedef (t_typedef* ttypedef); + void generate_enum (t_enum* tenum); + void generate_const (t_const* tconst); + void generate_struct (t_struct* tstruct); + void generate_xception (t_struct* txception); + void generate_service (t_service* tservice); + + std::string render_const_value(t_type* type, t_const_value* value); + + /** + * Struct generation code + */ + + void generate_py_struct(t_struct* tstruct, bool is_exception); + void generate_py_struct_definition(std::ofstream& out, t_struct* tstruct, bool is_xception=false, bool is_result=false); + void generate_py_struct_reader(std::ofstream& out, t_struct* tstruct); + void generate_py_struct_writer(std::ofstream& out, t_struct* tstruct); + void generate_py_function_helpers(t_function* tfunction); + + /** + * Service-level generation functions + */ + + void generate_service_helpers (t_service* tservice); + void generate_service_interface (t_service* tservice); + void generate_service_client (t_service* tservice); + void generate_service_remote (t_service* tservice); + void generate_service_server (t_service* tservice); + void generate_process_function (t_service* tservice, t_function* tfunction); + + /** + * Serialization constructs + */ + + void generate_deserialize_field (std::ofstream &out, + t_field* tfield, + std::string prefix="", + bool inclass=false); + + void generate_deserialize_struct (std::ofstream &out, + t_struct* tstruct, + std::string prefix=""); + + void generate_deserialize_container (std::ofstream &out, + t_type* ttype, + std::string prefix=""); + + void generate_deserialize_set_element (std::ofstream &out, + t_set* tset, + std::string prefix=""); + + void generate_deserialize_map_element (std::ofstream &out, + t_map* tmap, + std::string prefix=""); + + void generate_deserialize_list_element (std::ofstream &out, + t_list* tlist, + std::string prefix=""); + + void generate_serialize_field (std::ofstream &out, + t_field* tfield, + std::string prefix=""); + + void generate_serialize_struct (std::ofstream &out, + t_struct* tstruct, + std::string prefix=""); + + void generate_serialize_container (std::ofstream &out, + t_type* ttype, + std::string prefix=""); + + void generate_serialize_map_element (std::ofstream &out, + t_map* tmap, + std::string kiter, + std::string viter); + + void generate_serialize_set_element (std::ofstream &out, + t_set* tmap, + std::string iter); + + void generate_serialize_list_element (std::ofstream &out, + t_list* tlist, + std::string iter); + + void generate_python_docstring (std::ofstream& out, + t_struct* tstruct); + + void generate_python_docstring (std::ofstream& out, + t_function* tfunction); + + void generate_python_docstring (std::ofstream& out, + t_doc* tdoc, + t_struct* tstruct, + const char* subheader); + + void generate_python_docstring (std::ofstream& out, + t_doc* tdoc); + + /** + * Helper rendering functions + */ + + std::string py_autogen_comment(); + std::string py_imports(); + std::string render_includes(); + std::string render_fastbinary_includes(); + std::string declare_argument(t_field* tfield); + std::string render_field_default_value(t_field* tfield); + std::string type_name(t_type* ttype); + std::string function_signature(t_function* tfunction, std::string prefix=""); + std::string function_signature_if(t_function* tfunction, std::string prefix=""); + std::string argument_list(t_struct* tstruct); + std::string type_to_enum(t_type* ttype); + std::string type_to_spec_args(t_type* ttype); + + static std::string get_real_py_module(const t_program* program) { + std::string real_module = program->get_namespace("py"); + if (real_module.empty()) { + return program->get_name(); + } + return real_module; + } + + private: + + /** + * True iff we should generate new-style classes. + */ + bool gen_newstyle_; + + /** + * True iff we should generate Twisted-friendly RPC services. + */ + bool gen_twisted_; + + /** + * File streams + */ + + std::ofstream f_types_; + std::ofstream f_consts_; + std::ofstream f_service_; + + std::string package_dir_; + +}; + + +/** + * Prepares for file generation by opening up the necessary file output + * streams. + * + * @param tprogram The program to generate + */ +void t_py_generator::init_generator() { + // Make output directory + string module = get_real_py_module(program_); + package_dir_ = get_out_dir(); + while (true) { + // TODO: Do better error checking here. + MKDIR(package_dir_.c_str()); + std::ofstream init_py((package_dir_+"/__init__.py").c_str()); + init_py.close(); + if (module.empty()) { + break; + } + string::size_type pos = module.find('.'); + if (pos == string::npos) { + package_dir_ += "/"; + package_dir_ += module; + module.clear(); + } else { + package_dir_ += "/"; + package_dir_ += module.substr(0, pos); + module.erase(0, pos+1); + } + } + + // Make output file + string f_types_name = package_dir_+"/"+"ttypes.py"; + f_types_.open(f_types_name.c_str()); + + string f_consts_name = package_dir_+"/"+"constants.py"; + f_consts_.open(f_consts_name.c_str()); + + string f_init_name = package_dir_+"/__init__.py"; + ofstream f_init; + f_init.open(f_init_name.c_str()); + f_init << + "__all__ = ['ttypes', 'constants'"; + vector services = program_->get_services(); + vector::iterator sv_iter; + for (sv_iter = services.begin(); sv_iter != services.end(); ++sv_iter) { + f_init << ", '" << (*sv_iter)->get_name() << "'"; + } + f_init << "]" << endl; + f_init.close(); + + // Print header + f_types_ << + py_autogen_comment() << endl << + py_imports() << endl << + render_includes() << endl << + render_fastbinary_includes() << + endl << endl; + + f_consts_ << + py_autogen_comment() << endl << + py_imports() << endl << + "from ttypes import *" << endl << + endl; +} + +/** + * Renders all the imports necessary for including another Thrift program + */ +string t_py_generator::render_includes() { + const vector& includes = program_->get_includes(); + string result = ""; + for (size_t i = 0; i < includes.size(); ++i) { + result += "import " + get_real_py_module(includes[i]) + ".ttypes\n"; + } + if (includes.size() > 0) { + result += "\n"; + } + return result; +} + +/** + * Renders all the imports necessary to use the accelerated TBinaryProtocol + */ +string t_py_generator::render_fastbinary_includes() { + return + "from thrift.transport import TTransport\n" + "from thrift.protocol import TBinaryProtocol\n" + "try:\n" + " from thrift.protocol import fastbinary\n" + "except:\n" + " fastbinary = None\n"; +} + +/** + * Autogen'd comment + */ +string t_py_generator::py_autogen_comment() { + return + std::string("#\n") + + "# Autogenerated by Thrift\n" + + "#\n" + + "# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n" + + "#\n"; +} + +/** + * Prints standard thrift imports + */ +string t_py_generator::py_imports() { + return + string("from thrift.Thrift import *"); +} + +/** + * Closes the type files + */ +void t_py_generator::close_generator() { + // Close types file + f_types_.close(); + f_consts_.close(); +} + +/** + * Generates a typedef. This is not done in Python, types are all implicit. + * + * @param ttypedef The type definition + */ +void t_py_generator::generate_typedef(t_typedef* ttypedef) {} + +/** + * Generates code for an enumerated type. Done using a class to scope + * the values. + * + * @param tenum The enumeration + */ +void t_py_generator::generate_enum(t_enum* tenum) { + f_types_ << + "class " << tenum->get_name() << + (gen_newstyle_ ? "(object)" : "") << + ":" << endl; + indent_up(); + generate_python_docstring(f_types_, tenum); + + vector constants = tenum->get_constants(); + vector::iterator c_iter; + int value = -1; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + + f_types_ << + indent() << (*c_iter)->get_name() << " = " << value << endl; + } + + indent_down(); + f_types_ << endl; +} + +/** + * Generate a constant value + */ +void t_py_generator::generate_const(t_const* tconst) { + t_type* type = tconst->get_type(); + string name = tconst->get_name(); + t_const_value* value = tconst->get_value(); + + indent(f_consts_) << name << " = " << render_const_value(type, value); + f_consts_ << endl << endl; +} + +/** + * Prints the value of a constant with the given type. Note that type checking + * is NOT performed in this function as it is always run beforehand using the + * validate_types method in main.cc + */ +string t_py_generator::render_const_value(t_type* type, t_const_value* value) { + type = get_true_type(type); + std::ostringstream out; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + out << '"' << get_escaped_string(value) << '"'; + break; + case t_base_type::TYPE_BOOL: + out << (value->get_integer() > 0 ? "True" : "False"); + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + out << value->get_integer(); + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() == t_const_value::CV_INTEGER) { + out << value->get_integer(); + } else { + out << value->get_double(); + } + break; + default: + throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + indent(out) << value->get_integer(); + } else if (type->is_struct() || type->is_xception()) { + out << type->get_name() << "(**{" << endl; + indent_up(); + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + out << indent(); + out << render_const_value(g_type_string, v_iter->first); + out << " : "; + out << render_const_value(field_type, v_iter->second); + out << "," << endl; + } + indent_down(); + indent(out) << "})"; + } else if (type->is_map()) { + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + out << "{" << endl; + indent_up(); + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + out << indent(); + out << render_const_value(ktype, v_iter->first); + out << " : "; + out << render_const_value(vtype, v_iter->second); + out << "," << endl; + } + indent_down(); + indent(out) << "}"; + } else if (type->is_list() || type->is_set()) { + t_type* etype; + if (type->is_list()) { + etype = ((t_list*)type)->get_elem_type(); + } else { + etype = ((t_set*)type)->get_elem_type(); + } + if (type->is_set()) { + out << "set("; + } + out << "[" << endl; + indent_up(); + const vector& val = value->get_list(); + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + out << indent(); + out << render_const_value(etype, *v_iter); + out << "," << endl; + } + indent_down(); + indent(out) << "]"; + if (type->is_set()) { + out << ")"; + } + } else { + throw "CANNOT GENERATE CONSTANT FOR TYPE: " + type->get_name(); + } + + return out.str(); +} + +/** + * Generates a python struct + */ +void t_py_generator::generate_struct(t_struct* tstruct) { + generate_py_struct(tstruct, false); +} + +/** + * Generates a struct definition for a thrift exception. Basically the same + * as a struct but extends the Exception class. + * + * @param txception The struct definition + */ +void t_py_generator::generate_xception(t_struct* txception) { + generate_py_struct(txception, true); +} + +/** + * Generates a python struct + */ +void t_py_generator::generate_py_struct(t_struct* tstruct, + bool is_exception) { + generate_py_struct_definition(f_types_, tstruct, is_exception); +} + +/** + * Generates a struct definition for a thrift data type. + * + * @param tstruct The struct definition + */ +void t_py_generator::generate_py_struct_definition(ofstream& out, + t_struct* tstruct, + bool is_exception, + bool is_result) { + + const vector& members = tstruct->get_members(); + const vector& sorted_members = tstruct->get_sorted_members(); + vector::const_iterator m_iter; + + out << + "class " << tstruct->get_name(); + if (is_exception) { + out << "(Exception)"; + } else if (gen_newstyle_) { + out << "(object)"; + } + out << + ":" << endl; + indent_up(); + generate_python_docstring(out, tstruct); + + out << endl; + + /* + Here we generate the structure specification for the fastbinary codec. + These specifications have the following structure: + thrift_spec -> tuple of item_spec + item_spec -> None | (tag, type_enum, name, spec_args, default) + tag -> integer + type_enum -> TType.I32 | TType.STRING | TType.STRUCT | ... + name -> string_literal + default -> None # Handled by __init__ + spec_args -> None # For simple types + | (type_enum, spec_args) # Value type for list/set + | (type_enum, spec_args, type_enum, spec_args) + # Key and value for map + | (class_name, spec_args_ptr) # For struct/exception + class_name -> identifier # Basically a pointer to the class + spec_args_ptr -> expression # just class_name.spec_args + + TODO(dreiss): Consider making this work for structs with negative tags. + */ + + // TODO(dreiss): Look into generating an empty tuple instead of None + // for structures with no members. + // TODO(dreiss): Test encoding of structs where some inner structs + // don't have thrift_spec. + if (sorted_members.empty() || (sorted_members[0]->get_key() >= 0)) { + indent(out) << "thrift_spec = (" << endl; + indent_up(); + + int sorted_keys_pos = 0; + for (m_iter = sorted_members.begin(); m_iter != sorted_members.end(); ++m_iter) { + + for (; sorted_keys_pos != (*m_iter)->get_key(); sorted_keys_pos++) { + indent(out) << "None, # " << sorted_keys_pos << endl; + } + + indent(out) << "(" << (*m_iter)->get_key() << ", " + << type_to_enum((*m_iter)->get_type()) << ", " + << "'" << (*m_iter)->get_name() << "'" << ", " + << type_to_spec_args((*m_iter)->get_type()) << ", " + << render_field_default_value(*m_iter) << ", " + << ")," + << " # " << sorted_keys_pos + << endl; + + sorted_keys_pos ++; + } + + indent_down(); + indent(out) << ")" << endl << endl; + } else { + indent(out) << "thrift_spec = None" << endl; + } + + + if (members.size() > 0) { + out << + indent() << "def __init__(self,"; + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + // This fills in default values, as opposed to nulls + out << " " << declare_argument(*m_iter) << ","; + } + + out << "):" << endl; + + indent_up(); + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + // Initialize fields + t_type* type = (*m_iter)->get_type(); + if (!type->is_base_type() && !type->is_enum() && (*m_iter)->get_value() != NULL) { + indent(out) << + "if " << (*m_iter)->get_name() << " is " << "self.thrift_spec[" << + (*m_iter)->get_key() << "][4]:" << endl; + indent(out) << " " << (*m_iter)->get_name() << " = " << + render_field_default_value(*m_iter) << endl; + } + indent(out) << + "self." << (*m_iter)->get_name() << " = " << (*m_iter)->get_name() << endl; + } + + indent_down(); + + out << endl; + } + + generate_py_struct_reader(out, tstruct); + generate_py_struct_writer(out, tstruct); + + // For exceptions only, generate a __str__ method. This is + // because when raised exceptions are printed to the console, __repr__ + // isn't used. See python bug #5882 + if (is_exception) { + out << + indent() << "def __str__(self):" << endl << + indent() << " return repr(self)" << endl << + endl; + } + + // Printing utilities so that on the command line thrift + // structs look pretty like dictionaries + out << + indent() << "def __repr__(self):" << endl << + indent() << " L = ['%s=%r' % (key, value)" << endl << + indent() << " for key, value in self.__dict__.iteritems()]" << endl << + indent() << " return '%s(%s)' % (self.__class__.__name__, ', '.join(L))" << endl << + endl; + + // Equality and inequality methods that compare by value + out << + indent() << "def __eq__(self, other):" << endl; + indent_up(); + out << + indent() << "return isinstance(other, self.__class__) and " + "self.__dict__ == other.__dict__" << endl; + indent_down(); + out << endl; + + out << + indent() << "def __ne__(self, other):" << endl; + indent_up(); + out << + indent() << "return not (self == other)" << endl; + indent_down(); + out << endl; + + indent_down(); +} + +/** + * Generates the read method for a struct + */ +void t_py_generator::generate_py_struct_reader(ofstream& out, + t_struct* tstruct) { + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + indent(out) << + "def read(self, iprot):" << endl; + indent_up(); + + indent(out) << + "if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated " + "and isinstance(iprot.trans, TTransport.CReadableTransport) " + "and self.thrift_spec is not None " + "and fastbinary is not None:" << endl; + indent_up(); + + indent(out) << + "fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))" << endl; + indent(out) << + "return" << endl; + indent_down(); + + indent(out) << + "iprot.readStructBegin()" << endl; + + // Loop over reading in fields + indent(out) << + "while True:" << endl; + indent_up(); + + // Read beginning field marker + indent(out) << + "(fname, ftype, fid) = iprot.readFieldBegin()" << endl; + + // Check for field STOP marker and break + indent(out) << + "if ftype == TType.STOP:" << endl; + indent_up(); + indent(out) << + "break" << endl; + indent_down(); + + // Switch statement on the field we are reading + bool first = true; + + // Generate deserialization code for known cases + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + out << + indent() << "if "; + } else { + out << + indent() << "elif "; + } + out << "fid == " << (*f_iter)->get_key() << ":" << endl; + indent_up(); + indent(out) << "if ftype == " << type_to_enum((*f_iter)->get_type()) << ":" << endl; + indent_up(); + generate_deserialize_field(out, *f_iter, "self."); + indent_down(); + out << + indent() << "else:" << endl << + indent() << " iprot.skip(ftype)" << endl; + indent_down(); + } + + // In the default case we skip the field + out << + indent() << "else:" << endl << + indent() << " iprot.skip(ftype)" << endl; + + // Read field end marker + indent(out) << + "iprot.readFieldEnd()" << endl; + + indent_down(); + + indent(out) << + "iprot.readStructEnd()" << endl; + + indent_down(); + out << endl; +} + +void t_py_generator::generate_py_struct_writer(ofstream& out, + t_struct* tstruct) { + string name = tstruct->get_name(); + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + indent(out) << + "def write(self, oprot):" << endl; + indent_up(); + + indent(out) << + "if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated " + "and self.thrift_spec is not None " + "and fastbinary is not None:" << endl; + indent_up(); + + indent(out) << + "oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))" << endl; + indent(out) << + "return" << endl; + indent_down(); + + indent(out) << + "oprot.writeStructBegin('" << name << "')" << endl; + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + // Write field header + indent(out) << + "if self." << (*f_iter)->get_name() << " != None:" << endl; + indent_up(); + indent(out) << + "oprot.writeFieldBegin(" << + "'" << (*f_iter)->get_name() << "', " << + type_to_enum((*f_iter)->get_type()) << ", " << + (*f_iter)->get_key() << ")" << endl; + + // Write field contents + generate_serialize_field(out, *f_iter, "self."); + + // Write field closer + indent(out) << + "oprot.writeFieldEnd()" << endl; + + indent_down(); + } + + // Write the struct map + out << + indent() << "oprot.writeFieldStop()" << endl << + indent() << "oprot.writeStructEnd()" << endl; + + indent_down(); + out << + endl; +} + +/** + * Generates a thrift service. + * + * @param tservice The service definition + */ +void t_py_generator::generate_service(t_service* tservice) { + string f_service_name = package_dir_+"/"+service_name_+".py"; + f_service_.open(f_service_name.c_str()); + + f_service_ << + py_autogen_comment() << endl << + py_imports() << endl; + + if (tservice->get_extends() != NULL) { + f_service_ << + "import " << get_real_py_module(tservice->get_extends()->get_program()) << + "." << tservice->get_extends()->get_name() << endl; + } + + f_service_ << + "from ttypes import *" << endl << + "from thrift.Thrift import TProcessor" << endl << + render_fastbinary_includes() << endl; + + if (gen_twisted_) { + f_service_ << + "from zope.interface import Interface, implements" << endl << + "from twisted.internet import defer" << endl << + "from thrift.transport import TTwisted" << endl; + } + + f_service_ << endl; + + // Generate the three main parts of the service (well, two for now in PHP) + generate_service_interface(tservice); + generate_service_client(tservice); + generate_service_server(tservice); + generate_service_helpers(tservice); + generate_service_remote(tservice); + + // Close service file + f_service_ << endl; + f_service_.close(); +} + +/** + * Generates helper functions for a service. + * + * @param tservice The service to generate a header definition for + */ +void t_py_generator::generate_service_helpers(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + f_service_ << + "# HELPER FUNCTIONS AND STRUCTURES" << endl << endl; + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* ts = (*f_iter)->get_arglist(); + generate_py_struct_definition(f_service_, ts, false); + generate_py_function_helpers(*f_iter); + } +} + +/** + * Generates a struct and helpers for a function. + * + * @param tfunction The function + */ +void t_py_generator::generate_py_function_helpers(t_function* tfunction) { + if (!tfunction->is_oneway()) { + t_struct result(program_, tfunction->get_name() + "_result"); + t_field success(tfunction->get_returntype(), "success", 0); + if (!tfunction->get_returntype()->is_void()) { + result.append(&success); + } + + t_struct* xs = tfunction->get_xceptions(); + const vector& fields = xs->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + result.append(*f_iter); + } + generate_py_struct_definition(f_service_, &result, false, true); + } +} + +/** + * Generates a service interface definition. + * + * @param tservice The service to generate a header definition for + */ +void t_py_generator::generate_service_interface(t_service* tservice) { + string extends = ""; + string extends_if = ""; + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + extends_if = "(" + extends + ".Iface)"; + } else { + if (gen_twisted_) { + extends_if = "(Interface)"; + } + } + + f_service_ << + "class Iface" << extends_if << ":" << endl; + indent_up(); + generate_python_docstring(f_service_, tservice); + vector functions = tservice->get_functions(); + if (functions.empty()) { + f_service_ << + indent() << "pass" << endl; + } else { + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_service_ << + indent() << "def " << function_signature_if(*f_iter) << ":" << endl; + indent_up(); + generate_python_docstring(f_service_, (*f_iter)); + f_service_ << + indent() << "pass" << endl << endl; + indent_down(); + } + } + + indent_down(); + f_service_ << + endl; +} + +/** + * Generates a service client definition. + * + * @param tservice The service to generate a server for. + */ +void t_py_generator::generate_service_client(t_service* tservice) { + string extends = ""; + string extends_client = ""; + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + if (gen_twisted_) { + extends_client = "(" + extends + ".Client)"; + } else { + extends_client = extends + ".Client, "; + } + } else { + if (gen_twisted_ && gen_newstyle_) { + extends_client = "(object)"; + } + } + + if (gen_twisted_) { + f_service_ << + "class Client" << extends_client << ":" << endl << + " implements(Iface)" << endl << endl; + } else { + f_service_ << + "class Client(" << extends_client << "Iface):" << endl; + } + indent_up(); + generate_python_docstring(f_service_, tservice); + + // Constructor function + if (gen_twisted_) { + f_service_ << + indent() << "def __init__(self, transport, oprot_factory):" << endl; + } else { + f_service_ << + indent() << "def __init__(self, iprot, oprot=None):" << endl; + } + if (extends.empty()) { + if (gen_twisted_) { + f_service_ << + indent() << " self._transport = transport" << endl << + indent() << " self._oprot_factory = oprot_factory" << endl << + indent() << " self._seqid = 0" << endl << + indent() << " self._reqs = {}" << endl << + endl; + } else { + f_service_ << + indent() << " self._iprot = self._oprot = iprot" << endl << + indent() << " if oprot != None:" << endl << + indent() << " self._oprot = oprot" << endl << + indent() << " self._seqid = 0" << endl << + endl; + } + } else { + if (gen_twisted_) { + f_service_ << + indent() << " " << extends << ".Client.__init__(self, transport, oprot_factory)" << endl << + endl; + } else { + f_service_ << + indent() << " " << extends << ".Client.__init__(self, iprot, oprot)" << endl << + endl; + } + } + + // Generate client method implementations + vector functions = tservice->get_functions(); + vector::const_iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* arg_struct = (*f_iter)->get_arglist(); + const vector& fields = arg_struct->get_members(); + vector::const_iterator fld_iter; + string funname = (*f_iter)->get_name(); + + // Open function + indent(f_service_) << + "def " << function_signature(*f_iter) << ":" << endl; + indent_up(); + generate_python_docstring(f_service_, (*f_iter)); + if (gen_twisted_) { + indent(f_service_) << "self._seqid += 1" << endl; + if (!(*f_iter)->is_oneway()) { + indent(f_service_) << + "d = self._reqs[self._seqid] = defer.Deferred()" << endl; + } + } + + indent(f_service_) << + "self.send_" << funname << "("; + + bool first = true; + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << (*fld_iter)->get_name(); + } + f_service_ << ")" << endl; + + if (!(*f_iter)->is_oneway()) { + f_service_ << indent(); + if (gen_twisted_) { + f_service_ << "return d" << endl; + } else { + if (!(*f_iter)->get_returntype()->is_void()) { + f_service_ << "return "; + } + f_service_ << + "self.recv_" << funname << "()" << endl; + } + } else { + if (gen_twisted_) { + f_service_ << + indent() << "return defer.succeed(None)" << endl; + } + } + indent_down(); + f_service_ << endl; + + indent(f_service_) << + "def send_" << function_signature(*f_iter) << ":" << endl; + + indent_up(); + + std::string argsname = (*f_iter)->get_name() + "_args"; + + // Serialize the request header + if (gen_twisted_) { + f_service_ << + indent() << "oprot = self._oprot_factory.getProtocol(self._transport)" << endl << + indent() << + "oprot.writeMessageBegin('" << (*f_iter)->get_name() << "', TMessageType.CALL, self._seqid)" + << endl; + } else { + f_service_ << + indent() << "self._oprot.writeMessageBegin('" << (*f_iter)->get_name() << "', TMessageType.CALL, self._seqid)" << endl; + } + + f_service_ << + indent() << "args = " << argsname << "()" << endl; + + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + f_service_ << + indent() << "args." << (*fld_iter)->get_name() << " = " << (*fld_iter)->get_name() << endl; + } + + // Write to the stream + if (gen_twisted_) { + f_service_ << + indent() << "args.write(oprot)" << endl << + indent() << "oprot.writeMessageEnd()" << endl << + indent() << "oprot.trans.flush()" << endl; + } else { + f_service_ << + indent() << "args.write(self._oprot)" << endl << + indent() << "self._oprot.writeMessageEnd()" << endl << + indent() << "self._oprot.trans.flush()" << endl; + } + + indent_down(); + + if (!(*f_iter)->is_oneway()) { + std::string resultname = (*f_iter)->get_name() + "_result"; + // Open function + f_service_ << + endl; + if (gen_twisted_) { + f_service_ << + indent() << "def recv_" << (*f_iter)->get_name() << + "(self, iprot, mtype, rseqid):" << endl; + } else { + t_struct noargs(program_); + t_function recv_function((*f_iter)->get_returntype(), + string("recv_") + (*f_iter)->get_name(), + &noargs); + f_service_ << + indent() << "def " << function_signature(&recv_function) << ":" << endl; + } + indent_up(); + + // TODO(mcslee): Validate message reply here, seq ids etc. + + if (gen_twisted_) { + f_service_ << + indent() << "d = self._reqs.pop(rseqid)" << endl; + } else { + f_service_ << + indent() << "(fname, mtype, rseqid) = self._iprot.readMessageBegin()" << endl; + } + + f_service_ << + indent() << "if mtype == TMessageType.EXCEPTION:" << endl << + indent() << " x = TApplicationException()" << endl; + + if (gen_twisted_) { + f_service_ << + indent() << " x.read(iprot)" << endl << + indent() << " iprot.readMessageEnd()" << endl << + indent() << " return d.errback(x)" << endl << + indent() << "result = " << resultname << "()" << endl << + indent() << "result.read(iprot)" << endl << + indent() << "iprot.readMessageEnd()" << endl; + } else { + f_service_ << + indent() << " x.read(self._iprot)" << endl << + indent() << " self._iprot.readMessageEnd()" << endl << + indent() << " raise x" << endl << + indent() << "result = " << resultname << "()" << endl << + indent() << "result.read(self._iprot)" << endl << + indent() << "self._iprot.readMessageEnd()" << endl; + } + + // Careful, only return _result if not a void function + if (!(*f_iter)->get_returntype()->is_void()) { + f_service_ << + indent() << "if result.success != None:" << endl; + if (gen_twisted_) { + f_service_ << + indent() << " return d.callback(result.success)" << endl; + } else { + f_service_ << + indent() << " return result.success" << endl; + } + } + + t_struct* xs = (*f_iter)->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << + indent() << "if result." << (*x_iter)->get_name() << " != None:" << endl; + if (gen_twisted_) { + f_service_ << + indent() << " return d.errback(result." << (*x_iter)->get_name() << ")" << endl; + + } else { + f_service_ << + indent() << " raise result." << (*x_iter)->get_name() << "" << endl; + } + } + + // Careful, only return _result if not a void function + if ((*f_iter)->get_returntype()->is_void()) { + if (gen_twisted_) { + indent(f_service_) << + "return d.callback(None)" << endl; + } else { + indent(f_service_) << + "return" << endl; + } + } else { + if (gen_twisted_) { + f_service_ << + indent() << "return d.errback(TApplicationException(TApplicationException.MISSING_RESULT, \"" << (*f_iter)->get_name() << " failed: unknown result\"))" << endl; + } else { + f_service_ << + indent() << "raise TApplicationException(TApplicationException.MISSING_RESULT, \"" << (*f_iter)->get_name() << " failed: unknown result\");" << endl; + } + } + + // Close function + indent_down(); + f_service_ << endl; + } + } + + indent_down(); + f_service_ << + endl; +} + +/** + * Generates a command line tool for making remote requests + * + * @param tservice The service to generate a remote for. + */ +void t_py_generator::generate_service_remote(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + string f_remote_name = package_dir_+"/"+service_name_+"-remote"; + ofstream f_remote; + f_remote.open(f_remote_name.c_str()); + + f_remote << + "#!/usr/bin/env python" << endl << + py_autogen_comment() << endl << + "import sys" << endl << + "import pprint" << endl << + "from urlparse import urlparse" << endl << + "from thrift.transport import TTransport" << endl << + "from thrift.transport import TSocket" << endl << + "from thrift.transport import THttpClient" << endl << + "from thrift.protocol import TBinaryProtocol" << endl << + endl; + + f_remote << + "import " << service_name_ << endl << + "from ttypes import *" << endl << + endl; + + f_remote << + "if len(sys.argv) <= 1 or sys.argv[1] == '--help':" << endl << + " print ''" << endl << + " print 'Usage: ' + sys.argv[0] + ' [-h host:port] [-u url] [-f[ramed]] function [arg1 [arg2...]]'" << endl << + " print ''" << endl << + " print 'Functions:'" << endl; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_remote << + " print ' " << (*f_iter)->get_returntype()->get_name() << " " << (*f_iter)->get_name() << "("; + t_struct* arg_struct = (*f_iter)->get_arglist(); + const std::vector& args = arg_struct->get_members(); + vector::const_iterator a_iter; + int num_args = args.size(); + bool first = true; + for (int i = 0; i < num_args; ++i) { + if (first) { + first = false; + } else { + f_remote << ", "; + } + f_remote << + args[i]->get_type()->get_name() << " " << args[i]->get_name(); + } + f_remote << ")'" << endl; + } + f_remote << + " print ''" << endl << + " sys.exit(0)" << endl << + endl; + + f_remote << + "pp = pprint.PrettyPrinter(indent = 2)" << endl << + "host = 'localhost'" << endl << + "port = 9090" << endl << + "uri = ''" << endl << + "framed = False" << endl << + "http = False" << endl << + "argi = 1" << endl << + endl << + "if sys.argv[argi] == '-h':" << endl << + " parts = sys.argv[argi+1].split(':') " << endl << + " host = parts[0]" << endl << + " port = int(parts[1])" << endl << + " argi += 2" << endl << + endl << + "if sys.argv[argi] == '-u':" << endl << + " url = urlparse(sys.argv[argi+1])" << endl << + " parts = url[1].split(':') " << endl << + " host = parts[0]" << endl << + " if len(parts) > 1:" << endl << + " port = int(parts[1])" << endl << + " else:" << endl << + " port = 80" << endl << + " uri = url[2]" << endl << + " http = True" << endl << + " argi += 2" << endl << + endl << + "if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed':" << endl << + " framed = True" << endl << + " argi += 1" << endl << + endl << + "cmd = sys.argv[argi]" << endl << + "args = sys.argv[argi+1:]" << endl << + endl << + "if http:" << endl << + " transport = THttpClient.THttpClient(host, port, uri)" << endl << + "else:" << endl << + " socket = TSocket.TSocket(host, port)" << endl << + " if framed:" << endl << + " transport = TTransport.TFramedTransport(socket)" << endl << + " else:" << endl << + " transport = TTransport.TBufferedTransport(socket)" << endl << + "protocol = TBinaryProtocol.TBinaryProtocol(transport)" << endl << + "client = " << service_name_ << ".Client(protocol)" << endl << + "transport.open()" << endl << + endl; + + // Generate the dispatch methods + bool first = true; + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + if (first) { + first = false; + } else { + f_remote << "el"; + } + + t_struct* arg_struct = (*f_iter)->get_arglist(); + const std::vector& args = arg_struct->get_members(); + vector::const_iterator a_iter; + int num_args = args.size(); + + f_remote << + "if cmd == '" << (*f_iter)->get_name() << "':" << endl << + " if len(args) != " << num_args << ":" << endl << + " print '" << (*f_iter)->get_name() << " requires " << num_args << " args'" << endl << + " sys.exit(1)" << endl << + " pp.pprint(client." << (*f_iter)->get_name() << "("; + for (int i = 0; i < num_args; ++i) { + if (args[i]->get_type()->is_string()) { + f_remote << "args[" << i << "],"; + } else { + f_remote << "eval(args[" << i << "]),"; + } + } + f_remote << "))" << endl; + + f_remote << endl; + } + + f_remote << "transport.close()" << endl; + + // Close service file + f_remote.close(); + + // Make file executable, love that bitwise OR action + chmod(f_remote_name.c_str(), + S_IRUSR + | S_IWUSR + | S_IXUSR +#ifndef MINGW + | S_IRGRP + | S_IXGRP + | S_IROTH + | S_IXOTH +#endif + ); +} + +/** + * Generates a service server definition. + * + * @param tservice The service to generate a server for. + */ +void t_py_generator::generate_service_server(t_service* tservice) { + // Generate the dispatch methods + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + string extends = ""; + string extends_processor = ""; + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + extends_processor = extends + ".Processor, "; + } + + // Generate the header portion + if (gen_twisted_) { + f_service_ << + "class Processor(" << extends_processor << "TProcessor):" << endl << + " implements(Iface)" << endl << endl; + } else { + f_service_ << + "class Processor(" << extends_processor << "Iface, TProcessor):" << endl; + } + + indent_up(); + + indent(f_service_) << + "def __init__(self, handler):" << endl; + indent_up(); + if (extends.empty()) { + if (gen_twisted_) { + f_service_ << + indent() << "self._handler = Iface(handler)" << endl; + } else { + f_service_ << + indent() << "self._handler = handler" << endl; + } + + f_service_ << + indent() << "self._processMap = {}" << endl; + } else { + if (gen_twisted_) { + f_service_ << + indent() << extends << ".Processor.__init__(self, Iface(handler))" << endl; + } else { + f_service_ << + indent() << extends << ".Processor.__init__(self, handler)" << endl; + } + } + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_service_ << + indent() << "self._processMap[\"" << (*f_iter)->get_name() << "\"] = Processor.process_" << (*f_iter)->get_name() << endl; + } + indent_down(); + f_service_ << endl; + + // Generate the server implementation + indent(f_service_) << + "def process(self, iprot, oprot):" << endl; + indent_up(); + + f_service_ << + indent() << "(name, type, seqid) = iprot.readMessageBegin()" << endl; + + // TODO(mcslee): validate message + + // HOT: dictionary function lookup + f_service_ << + indent() << "if name not in self._processMap:" << endl << + indent() << " iprot.skip(TType.STRUCT)" << endl << + indent() << " iprot.readMessageEnd()" << endl << + indent() << " x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name))" << endl << + indent() << " oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid)" << endl << + indent() << " x.write(oprot)" << endl << + indent() << " oprot.writeMessageEnd()" << endl << + indent() << " oprot.trans.flush()" << endl; + + if (gen_twisted_) { + f_service_ << + indent() << " return defer.succeed(None)" << endl; + } else { + f_service_ << + indent() << " return" << endl; + } + + f_service_ << + indent() << "else:" << endl; + + if (gen_twisted_) { + f_service_ << + indent() << " return self._processMap[name](self, seqid, iprot, oprot)" << endl; + } else { + f_service_ << + indent() << " self._processMap[name](self, seqid, iprot, oprot)" << endl; + + // Read end of args field, the T_STOP, and the struct close + f_service_ << + indent() << "return True" << endl; + } + + indent_down(); + f_service_ << endl; + + // Generate the process subfunctions + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + generate_process_function(tservice, *f_iter); + } + + indent_down(); + f_service_ << endl; +} + +/** + * Generates a process function definition. + * + * @param tfunction The function to write a dispatcher for + */ +void t_py_generator::generate_process_function(t_service* tservice, + t_function* tfunction) { + // Open function + indent(f_service_) << + "def process_" << tfunction->get_name() << + "(self, seqid, iprot, oprot):" << endl; + indent_up(); + + string argsname = tfunction->get_name() + "_args"; + string resultname = tfunction->get_name() + "_result"; + + f_service_ << + indent() << "args = " << argsname << "()" << endl << + indent() << "args.read(iprot)" << endl << + indent() << "iprot.readMessageEnd()" << endl; + + t_struct* xs = tfunction->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + + // Declare result for non oneway function + if (!tfunction->is_oneway()) { + f_service_ << + indent() << "result = " << resultname << "()" << endl; + } + + if (gen_twisted_) { + // Generate the function call + t_struct* arg_struct = tfunction->get_arglist(); + const std::vector& fields = arg_struct->get_members(); + vector::const_iterator f_iter; + + f_service_ << + indent() << "d = defer.maybeDeferred(self._handler." << + tfunction->get_name() << ", "; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << "args." << (*f_iter)->get_name(); + } + f_service_ << ")" << endl; + + // Shortcut out here for oneway functions + if (tfunction->is_oneway()) { + f_service_ << + indent() << "return d" << endl; + indent_down(); + f_service_ << endl; + return; + } + + f_service_ << + indent() << + "d.addCallback(self.write_results_success_" << + tfunction->get_name() << ", result, seqid, oprot)" << endl; + + if (xceptions.size() > 0) { + f_service_ << + indent() << + "d.addErrback(self.write_results_exception_" << + tfunction->get_name() << ", result, seqid, oprot)" << endl; + } + + f_service_ << + indent() << "return d" << endl; + + indent_down(); + f_service_ << endl; + + indent(f_service_) << + "def write_results_success_" << tfunction->get_name() << + "(self, success, result, seqid, oprot):" << endl; + indent_up(); + f_service_ << + indent() << "result.success = success" << endl << + indent() << "oprot.writeMessageBegin(\"" << tfunction->get_name() << + "\", TMessageType.REPLY, seqid)" << endl << + indent() << "result.write(oprot)" << endl << + indent() << "oprot.writeMessageEnd()" << endl << + indent() << "oprot.trans.flush()" << endl; + indent_down(); + f_service_ << endl; + + // Try block for a function with exceptions + if (!tfunction->is_oneway() && xceptions.size() > 0) { + indent(f_service_) << + "def write_results_exception_" << tfunction->get_name() << + "(self, error, result, seqid, oprot):" << endl; + indent_up(); + f_service_ << + indent() << "try:" << endl; + + // Kinda absurd + f_service_ << + indent() << " error.raiseException()" << endl; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << + indent() << "except " << type_name((*x_iter)->get_type()) << ", " << (*x_iter)->get_name() << ":" << endl; + if (!tfunction->is_oneway()) { + indent_up(); + f_service_ << + indent() << "result." << (*x_iter)->get_name() << " = " << (*x_iter)->get_name() << endl; + indent_down(); + } else { + f_service_ << + indent() << "pass" << endl; + } + } + f_service_ << + indent() << "oprot.writeMessageBegin(\"" << tfunction->get_name() << + "\", TMessageType.REPLY, seqid)" << endl << + indent() << "result.write(oprot)" << endl << + indent() << "oprot.writeMessageEnd()" << endl << + indent() << "oprot.trans.flush()" << endl; + indent_down(); + f_service_ << endl; + } + } else { + + // Try block for a function with exceptions + if (xceptions.size() > 0) { + f_service_ << + indent() << "try:" << endl; + indent_up(); + } + + // Generate the function call + t_struct* arg_struct = tfunction->get_arglist(); + const std::vector& fields = arg_struct->get_members(); + vector::const_iterator f_iter; + + f_service_ << indent(); + if (!tfunction->is_oneway() && !tfunction->get_returntype()->is_void()) { + f_service_ << "result.success = "; + } + f_service_ << + "self._handler." << tfunction->get_name() << "("; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << "args." << (*f_iter)->get_name(); + } + f_service_ << ")" << endl; + + if (!tfunction->is_oneway() && xceptions.size() > 0) { + indent_down(); + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << + indent() << "except " << type_name((*x_iter)->get_type()) << ", " << (*x_iter)->get_name() << ":" << endl; + if (!tfunction->is_oneway()) { + indent_up(); + f_service_ << + indent() << "result." << (*x_iter)->get_name() << " = " << (*x_iter)->get_name() << endl; + indent_down(); + } else { + f_service_ << + indent() << "pass" << endl; + } + } + } + + // Shortcut out here for oneway functions + if (tfunction->is_oneway()) { + f_service_ << + indent() << "return" << endl; + indent_down(); + f_service_ << endl; + return; + } + + f_service_ << + indent() << "oprot.writeMessageBegin(\"" << tfunction->get_name() << "\", TMessageType.REPLY, seqid)" << endl << + indent() << "result.write(oprot)" << endl << + indent() << "oprot.writeMessageEnd()" << endl << + indent() << "oprot.trans.flush()" << endl; + + // Close function + indent_down(); + f_service_ << endl; + } +} + +/** + * Deserializes a field of any type. + */ +void t_py_generator::generate_deserialize_field(ofstream &out, + t_field* tfield, + string prefix, + bool inclass) { + t_type* type = get_true_type(tfield->get_type()); + + if (type->is_void()) { + throw "CANNOT GENERATE DESERIALIZE CODE FOR void TYPE: " + + prefix + tfield->get_name(); + } + + string name = prefix + tfield->get_name(); + + if (type->is_struct() || type->is_xception()) { + generate_deserialize_struct(out, + (t_struct*)type, + name); + } else if (type->is_container()) { + generate_deserialize_container(out, type, name); + } else if (type->is_base_type() || type->is_enum()) { + indent(out) << + name << " = iprot."; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "compiler error: cannot serialize void field in a struct: " + + name; + break; + case t_base_type::TYPE_STRING: + out << "readString();"; + break; + case t_base_type::TYPE_BOOL: + out << "readBool();"; + break; + case t_base_type::TYPE_BYTE: + out << "readByte();"; + break; + case t_base_type::TYPE_I16: + out << "readI16();"; + break; + case t_base_type::TYPE_I32: + out << "readI32();"; + break; + case t_base_type::TYPE_I64: + out << "readI64();"; + break; + case t_base_type::TYPE_DOUBLE: + out << "readDouble();"; + break; + default: + throw "compiler error: no PHP name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + out << "readI32();"; + } + out << endl; + + } else { + printf("DO NOT KNOW HOW TO DESERIALIZE FIELD '%s' TYPE '%s'\n", + tfield->get_name().c_str(), type->get_name().c_str()); + } +} + +/** + * Generates an unserializer for a struct, calling read() + */ +void t_py_generator::generate_deserialize_struct(ofstream &out, + t_struct* tstruct, + string prefix) { + out << + indent() << prefix << " = " << type_name(tstruct) << "()" << endl << + indent() << prefix << ".read(iprot)" << endl; +} + +/** + * Serialize a container by writing out the header followed by + * data and then a footer. + */ +void t_py_generator::generate_deserialize_container(ofstream &out, + t_type* ttype, + string prefix) { + string size = tmp("_size"); + string ktype = tmp("_ktype"); + string vtype = tmp("_vtype"); + string etype = tmp("_etype"); + + t_field fsize(g_type_i32, size); + t_field fktype(g_type_byte, ktype); + t_field fvtype(g_type_byte, vtype); + t_field fetype(g_type_byte, etype); + + // Declare variables, read header + if (ttype->is_map()) { + out << + indent() << prefix << " = {}" << endl << + indent() << "(" << ktype << ", " << vtype << ", " << size << " ) = iprot.readMapBegin() " << endl; + } else if (ttype->is_set()) { + out << + indent() << prefix << " = set()" << endl << + indent() << "(" << etype << ", " << size << ") = iprot.readSetBegin()" << endl; + } else if (ttype->is_list()) { + out << + indent() << prefix << " = []" << endl << + indent() << "(" << etype << ", " << size << ") = iprot.readListBegin()" << endl; + } + + // For loop iterates over elements + string i = tmp("_i"); + indent(out) << + "for " << i << " in xrange(" << size << "):" << endl; + + indent_up(); + + if (ttype->is_map()) { + generate_deserialize_map_element(out, (t_map*)ttype, prefix); + } else if (ttype->is_set()) { + generate_deserialize_set_element(out, (t_set*)ttype, prefix); + } else if (ttype->is_list()) { + generate_deserialize_list_element(out, (t_list*)ttype, prefix); + } + + indent_down(); + + // Read container end + if (ttype->is_map()) { + indent(out) << "iprot.readMapEnd()" << endl; + } else if (ttype->is_set()) { + indent(out) << "iprot.readSetEnd()" << endl; + } else if (ttype->is_list()) { + indent(out) << "iprot.readListEnd()" << endl; + } +} + + +/** + * Generates code to deserialize a map + */ +void t_py_generator::generate_deserialize_map_element(ofstream &out, + t_map* tmap, + string prefix) { + string key = tmp("_key"); + string val = tmp("_val"); + t_field fkey(tmap->get_key_type(), key); + t_field fval(tmap->get_val_type(), val); + + generate_deserialize_field(out, &fkey); + generate_deserialize_field(out, &fval); + + indent(out) << + prefix << "[" << key << "] = " << val << endl; +} + +/** + * Write a set element + */ +void t_py_generator::generate_deserialize_set_element(ofstream &out, + t_set* tset, + string prefix) { + string elem = tmp("_elem"); + t_field felem(tset->get_elem_type(), elem); + + generate_deserialize_field(out, &felem); + + indent(out) << + prefix << ".add(" << elem << ")" << endl; +} + +/** + * Write a list element + */ +void t_py_generator::generate_deserialize_list_element(ofstream &out, + t_list* tlist, + string prefix) { + string elem = tmp("_elem"); + t_field felem(tlist->get_elem_type(), elem); + + generate_deserialize_field(out, &felem); + + indent(out) << + prefix << ".append(" << elem << ")" << endl; +} + + +/** + * Serializes a field of any type. + * + * @param tfield The field to serialize + * @param prefix Name to prepend to field name + */ +void t_py_generator::generate_serialize_field(ofstream &out, + t_field* tfield, + string prefix) { + t_type* type = get_true_type(tfield->get_type()); + + // Do nothing for void types + if (type->is_void()) { + throw "CANNOT GENERATE SERIALIZE CODE FOR void TYPE: " + + prefix + tfield->get_name(); + } + + if (type->is_struct() || type->is_xception()) { + generate_serialize_struct(out, + (t_struct*)type, + prefix + tfield->get_name()); + } else if (type->is_container()) { + generate_serialize_container(out, + type, + prefix + tfield->get_name()); + } else if (type->is_base_type() || type->is_enum()) { + + string name = prefix + tfield->get_name(); + + indent(out) << + "oprot."; + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw + "compiler error: cannot serialize void field in a struct: " + name; + break; + case t_base_type::TYPE_STRING: + out << "writeString(" << name << ")"; + break; + case t_base_type::TYPE_BOOL: + out << "writeBool(" << name << ")"; + break; + case t_base_type::TYPE_BYTE: + out << "writeByte(" << name << ")"; + break; + case t_base_type::TYPE_I16: + out << "writeI16(" << name << ")"; + break; + case t_base_type::TYPE_I32: + out << "writeI32(" << name << ")"; + break; + case t_base_type::TYPE_I64: + out << "writeI64(" << name << ")"; + break; + case t_base_type::TYPE_DOUBLE: + out << "writeDouble(" << name << ")"; + break; + default: + throw "compiler error: no PHP name for base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + out << "writeI32(" << name << ")"; + } + out << endl; + } else { + printf("DO NOT KNOW HOW TO SERIALIZE FIELD '%s%s' TYPE '%s'\n", + prefix.c_str(), + tfield->get_name().c_str(), + type->get_name().c_str()); + } +} + +/** + * Serializes all the members of a struct. + * + * @param tstruct The struct to serialize + * @param prefix String prefix to attach to all fields + */ +void t_py_generator::generate_serialize_struct(ofstream &out, + t_struct* tstruct, + string prefix) { + indent(out) << + prefix << ".write(oprot)" << endl; +} + +void t_py_generator::generate_serialize_container(ofstream &out, + t_type* ttype, + string prefix) { + if (ttype->is_map()) { + indent(out) << + "oprot.writeMapBegin(" << + type_to_enum(((t_map*)ttype)->get_key_type()) << ", " << + type_to_enum(((t_map*)ttype)->get_val_type()) << ", " << + "len(" << prefix << "))" << endl; + } else if (ttype->is_set()) { + indent(out) << + "oprot.writeSetBegin(" << + type_to_enum(((t_set*)ttype)->get_elem_type()) << ", " << + "len(" << prefix << "))" << endl; + } else if (ttype->is_list()) { + indent(out) << + "oprot.writeListBegin(" << + type_to_enum(((t_list*)ttype)->get_elem_type()) << ", " << + "len(" << prefix << "))" << endl; + } + + if (ttype->is_map()) { + string kiter = tmp("kiter"); + string viter = tmp("viter"); + indent(out) << + "for " << kiter << "," << viter << " in " << prefix << ".items():" << endl; + indent_up(); + generate_serialize_map_element(out, (t_map*)ttype, kiter, viter); + indent_down(); + } else if (ttype->is_set()) { + string iter = tmp("iter"); + indent(out) << + "for " << iter << " in " << prefix << ":" << endl; + indent_up(); + generate_serialize_set_element(out, (t_set*)ttype, iter); + indent_down(); + } else if (ttype->is_list()) { + string iter = tmp("iter"); + indent(out) << + "for " << iter << " in " << prefix << ":" << endl; + indent_up(); + generate_serialize_list_element(out, (t_list*)ttype, iter); + indent_down(); + } + + if (ttype->is_map()) { + indent(out) << + "oprot.writeMapEnd()" << endl; + } else if (ttype->is_set()) { + indent(out) << + "oprot.writeSetEnd()" << endl; + } else if (ttype->is_list()) { + indent(out) << + "oprot.writeListEnd()" << endl; + } +} + +/** + * Serializes the members of a map. + * + */ +void t_py_generator::generate_serialize_map_element(ofstream &out, + t_map* tmap, + string kiter, + string viter) { + t_field kfield(tmap->get_key_type(), kiter); + generate_serialize_field(out, &kfield, ""); + + t_field vfield(tmap->get_val_type(), viter); + generate_serialize_field(out, &vfield, ""); +} + +/** + * Serializes the members of a set. + */ +void t_py_generator::generate_serialize_set_element(ofstream &out, + t_set* tset, + string iter) { + t_field efield(tset->get_elem_type(), iter); + generate_serialize_field(out, &efield, ""); +} + +/** + * Serializes the members of a list. + */ +void t_py_generator::generate_serialize_list_element(ofstream &out, + t_list* tlist, + string iter) { + t_field efield(tlist->get_elem_type(), iter); + generate_serialize_field(out, &efield, ""); +} + +/** + * Generates the docstring for a given struct. + */ +void t_py_generator::generate_python_docstring(ofstream& out, + t_struct* tstruct) { + generate_python_docstring(out, tstruct, tstruct, "Attributes"); +} + +/** + * Generates the docstring for a given function. + */ +void t_py_generator::generate_python_docstring(ofstream& out, + t_function* tfunction) { + generate_python_docstring(out, tfunction, tfunction->get_arglist(), "Parameters"); +} + +/** + * Generates the docstring for a struct or function. + */ +void t_py_generator::generate_python_docstring(ofstream& out, + t_doc* tdoc, + t_struct* tstruct, + const char* subheader) { + bool has_doc = false; + stringstream ss; + if (tdoc->has_doc()) { + has_doc = true; + ss << tdoc->get_doc(); + } + + const vector& fields = tstruct->get_members(); + if (fields.size() > 0) { + if (has_doc) { + ss << endl; + } + has_doc = true; + ss << subheader << ":\n"; + vector::const_iterator p_iter; + for (p_iter = fields.begin(); p_iter != fields.end(); ++p_iter) { + t_field* p = *p_iter; + ss << " - " << p->get_name(); + if (p->has_doc()) { + ss << ": " << p->get_doc(); + } else { + ss << endl; + } + } + } + + if (has_doc) { + generate_docstring_comment(out, + "\"\"\"\n", + "", ss.str(), + "\"\"\"\n"); + } +} + +/** + * Generates the docstring for a generic object. + */ +void t_py_generator::generate_python_docstring(ofstream& out, + t_doc* tdoc) { + if (tdoc->has_doc()) { + generate_docstring_comment(out, + "\"\"\"\n", + "", tdoc->get_doc(), + "\"\"\"\n"); + } +} + +/** + * Declares an argument, which may include initialization as necessary. + * + * @param tfield The field + */ +string t_py_generator::declare_argument(t_field* tfield) { + std::ostringstream result; + result << tfield->get_name() << "="; + if (tfield->get_value() != NULL) { + result << "thrift_spec[" << + tfield->get_key() << "][4]"; + } else { + result << "None"; + } + return result.str(); +} + +/** + * Renders a field default value, returns None otherwise. + * + * @param tfield The field + */ +string t_py_generator::render_field_default_value(t_field* tfield) { + t_type* type = get_true_type(tfield->get_type()); + if (tfield->get_value() != NULL) { + return render_const_value(type, tfield->get_value()); + } else { + return "None"; + } +} + +/** + * Renders a function signature of the form 'type name(args)' + * + * @param tfunction Function definition + * @return String of rendered function definition + */ +string t_py_generator::function_signature(t_function* tfunction, + string prefix) { + // TODO(mcslee): Nitpicky, no ',' if argument_list is empty + return + prefix + tfunction->get_name() + + "(self, " + argument_list(tfunction->get_arglist()) + ")"; +} + +/** + * Renders an interface function signature of the form 'type name(args)' + * + * @param tfunction Function definition + * @return String of rendered function definition + */ +string t_py_generator::function_signature_if(t_function* tfunction, + string prefix) { + // TODO(mcslee): Nitpicky, no ',' if argument_list is empty + string signature = prefix + tfunction->get_name() + "("; + if (!gen_twisted_) { + signature += "self, "; + } + signature += argument_list(tfunction->get_arglist()) + ")"; + return signature; +} + + +/** + * Renders a field list + */ +string t_py_generator::argument_list(t_struct* tstruct) { + string result = ""; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + result += ", "; + } + result += (*f_iter)->get_name(); + } + return result; +} + +string t_py_generator::type_name(t_type* ttype) { + t_program* program = ttype->get_program(); + if (ttype->is_service()) { + return get_real_py_module(program) + "." + ttype->get_name(); + } + if (program != NULL && program != program_) { + return get_real_py_module(program) + ".ttypes." + ttype->get_name(); + } + return ttype->get_name(); +} + +/** + * Converts the parse type to a Python tyoe + */ +string t_py_generator::type_to_enum(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + return "TType.STRING"; + case t_base_type::TYPE_BOOL: + return "TType.BOOL"; + case t_base_type::TYPE_BYTE: + return "TType.BYTE"; + case t_base_type::TYPE_I16: + return "TType.I16"; + case t_base_type::TYPE_I32: + return "TType.I32"; + case t_base_type::TYPE_I64: + return "TType.I64"; + case t_base_type::TYPE_DOUBLE: + return "TType.DOUBLE"; + } + } else if (type->is_enum()) { + return "TType.I32"; + } else if (type->is_struct() || type->is_xception()) { + return "TType.STRUCT"; + } else if (type->is_map()) { + return "TType.MAP"; + } else if (type->is_set()) { + return "TType.SET"; + } else if (type->is_list()) { + return "TType.LIST"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + +/** See the comment inside generate_py_struct_definition for what this is. */ +string t_py_generator::type_to_spec_args(t_type* ttype) { + while (ttype->is_typedef()) { + ttype = ((t_typedef*)ttype)->get_type(); + } + + if (ttype->is_base_type() || ttype->is_enum()) { + return "None"; + } else if (ttype->is_struct() || ttype->is_xception()) { + return "(" + type_name(ttype) + ", " + type_name(ttype) + ".thrift_spec)"; + } else if (ttype->is_map()) { + return "(" + + type_to_enum(((t_map*)ttype)->get_key_type()) + "," + + type_to_spec_args(((t_map*)ttype)->get_key_type()) + "," + + type_to_enum(((t_map*)ttype)->get_val_type()) + "," + + type_to_spec_args(((t_map*)ttype)->get_val_type()) + + ")"; + + } else if (ttype->is_set()) { + return "(" + + type_to_enum(((t_set*)ttype)->get_elem_type()) + "," + + type_to_spec_args(((t_set*)ttype)->get_elem_type()) + + ")"; + + } else if (ttype->is_list()) { + return "(" + + type_to_enum(((t_list*)ttype)->get_elem_type()) + "," + + type_to_spec_args(((t_list*)ttype)->get_elem_type()) + + ")"; + } + + throw "INVALID TYPE IN type_to_spec_args: " + ttype->get_name(); +} + + +THRIFT_REGISTER_GENERATOR(py, "Python", +" new_style: Generate new-style classes.\n" \ +" twisted: Generate Twisted-friendly RPC services.\n" +); diff --git a/compiler/cpp/src/generate/t_rb_generator.cc b/compiler/cpp/src/generate/t_rb_generator.cc new file mode 100644 index 00000000..708cd42a --- /dev/null +++ b/compiler/cpp/src/generate/t_rb_generator.cc @@ -0,0 +1,1097 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include "t_oop_generator.h" +#include "platform.h" +using namespace std; + + +/** + * Ruby code generator. + * + */ +class t_rb_generator : public t_oop_generator { + public: + t_rb_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_oop_generator(program) + { + out_dir_base_ = "gen-rb"; + } + + /** + * Init and close methods + */ + + void init_generator(); + void close_generator(); + + /** + * Program-level generation functions + */ + + void generate_typedef (t_typedef* ttypedef); + void generate_enum (t_enum* tenum); + void generate_const (t_const* tconst); + void generate_struct (t_struct* tstruct); + void generate_xception (t_struct* txception); + void generate_service (t_service* tservice); + + std::string render_const_value(t_type* type, t_const_value* value); + + /** + * Struct generation code + */ + + void generate_rb_struct(std::ofstream& out, t_struct* tstruct, bool is_exception); + void generate_rb_struct_required_validator(std::ofstream& out, t_struct* tstruct); + void generate_rb_function_helpers(t_function* tfunction); + void generate_rb_simple_constructor(std::ofstream& out, t_struct* tstruct); + void generate_rb_simple_exception_constructor(std::ofstream& out, t_struct* tstruct); + void generate_field_constants (std::ofstream& out, t_struct* tstruct); + void generate_accessors (std::ofstream& out, t_struct* tstruct); + void generate_field_defns (std::ofstream& out, t_struct* tstruct); + void generate_field_data (std::ofstream& out, t_type* field_type, const std::string& field_name, t_const_value* field_value, bool optional); + + /** + * Service-level generation functions + */ + + void generate_service_helpers (t_service* tservice); + void generate_service_interface (t_service* tservice); + void generate_service_client (t_service* tservice); + void generate_service_server (t_service* tservice); + void generate_process_function (t_service* tservice, t_function* tfunction); + + /** + * Serialization constructs + */ + + void generate_deserialize_field (std::ofstream &out, + t_field* tfield, + std::string prefix="", + bool inclass=false); + + void generate_deserialize_struct (std::ofstream &out, + t_struct* tstruct, + std::string prefix=""); + + void generate_deserialize_container (std::ofstream &out, + t_type* ttype, + std::string prefix=""); + + void generate_deserialize_set_element (std::ofstream &out, + t_set* tset, + std::string prefix=""); + + void generate_deserialize_map_element (std::ofstream &out, + t_map* tmap, + std::string prefix=""); + + void generate_deserialize_list_element (std::ofstream &out, + t_list* tlist, + std::string prefix=""); + + void generate_serialize_field (std::ofstream &out, + t_field* tfield, + std::string prefix=""); + + void generate_serialize_struct (std::ofstream &out, + t_struct* tstruct, + std::string prefix=""); + + void generate_serialize_container (std::ofstream &out, + t_type* ttype, + std::string prefix=""); + + void generate_serialize_map_element (std::ofstream &out, + t_map* tmap, + std::string kiter, + std::string viter); + + void generate_serialize_set_element (std::ofstream &out, + t_set* tmap, + std::string iter); + + void generate_serialize_list_element (std::ofstream &out, + t_list* tlist, + std::string iter); + + void generate_rdoc (std::ofstream& out, + t_doc* tdoc); + + /** + * Helper rendering functions + */ + + std::string rb_autogen_comment(); + std::string render_includes(); + std::string declare_field(t_field* tfield); + std::string type_name(t_type* ttype); + std::string full_type_name(t_type* ttype); + std::string function_signature(t_function* tfunction, std::string prefix=""); + std::string argument_list(t_struct* tstruct); + std::string type_to_enum(t_type* ttype); + + + + std::vector ruby_modules(t_program* p) { + std::string ns = p->get_namespace("rb"); + boost::tokenizer<> tok(ns); + std::vector modules; + + for(boost::tokenizer<>::iterator beg=tok.begin(); beg != tok.end(); ++beg) { + modules.push_back(capitalize(*beg)); + } + + return modules; + } + + void begin_namespace(std::ofstream&, std::vector); + void end_namespace(std::ofstream&, std::vector); + + private: + + /** + * File streams + */ + + std::ofstream f_types_; + std::ofstream f_consts_; + std::ofstream f_service_; + +}; + + +/** + * Prepares for file generation by opening up the necessary file output + * streams. + * + * @param tprogram The program to generate + */ +void t_rb_generator::init_generator() { + // Make output directory + MKDIR(get_out_dir().c_str()); + + // Make output file + string f_types_name = get_out_dir()+underscore(program_name_)+"_types.rb"; + f_types_.open(f_types_name.c_str()); + + string f_consts_name = get_out_dir()+underscore(program_name_)+"_constants.rb"; + f_consts_.open(f_consts_name.c_str()); + + // Print header + f_types_ << + rb_autogen_comment() << endl << + render_includes() << endl; + begin_namespace(f_types_, ruby_modules(program_)); + + f_consts_ << + rb_autogen_comment() << endl << + "require File.dirname(__FILE__) + '/" << underscore(program_name_) << "_types'" << endl << + endl; + begin_namespace(f_consts_, ruby_modules(program_)); + +} + +/** + * Renders all the imports necessary for including another Thrift program + */ +string t_rb_generator::render_includes() { + const vector& includes = program_->get_includes(); + string result = ""; + for (size_t i = 0; i < includes.size(); ++i) { + result += "require '" + underscore(includes[i]->get_name()) + "_types'\n"; + } + if (includes.size() > 0) { + result += "\n"; + } + return result; +} + +/** + * Autogen'd comment + */ +string t_rb_generator::rb_autogen_comment() { + return + std::string("#\n") + + "# Autogenerated by Thrift\n" + + "#\n" + + "# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n" + + "#\n"; +} + +/** + * Closes the type files + */ +void t_rb_generator::close_generator() { + // Close types file + end_namespace(f_types_, ruby_modules(program_)); + end_namespace(f_consts_, ruby_modules(program_)); + f_types_.close(); + f_consts_.close(); +} + +/** + * Generates a typedef. This is not done in Ruby, types are all implicit. + * + * @param ttypedef The type definition + */ +void t_rb_generator::generate_typedef(t_typedef* ttypedef) {} + +/** + * Generates code for an enumerated type. Done using a class to scope + * the values. + * + * @param tenum The enumeration + */ +void t_rb_generator::generate_enum(t_enum* tenum) { + indent(f_types_) << + "module " << capitalize(tenum->get_name()) << endl; + indent_up(); + + vector constants = tenum->get_constants(); + vector::iterator c_iter; + int value = -1; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + + // Ruby class constants have to be capitalized... omg i am so on the fence + // about languages strictly enforcing capitalization why can't we just all + // agree and play nice. + string name = capitalize((*c_iter)->get_name()); + + f_types_ << + indent() << name << " = " << value << endl; + } + + // Create a set with valid values for this enum + indent(f_types_) << "VALID_VALUES = Set.new(["; + bool first = true; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + // Populate the set + first ? first = false: f_types_ << ", "; + f_types_ << capitalize((*c_iter)->get_name()); + } + f_types_ << "]).freeze" << endl; + + indent_down(); + indent(f_types_) << + "end" << endl << endl; +} + +/** + * Generate a constant value + */ +void t_rb_generator::generate_const(t_const* tconst) { + t_type* type = tconst->get_type(); + string name = tconst->get_name(); + t_const_value* value = tconst->get_value(); + + name[0] = toupper(name[0]); + + indent(f_consts_) << name << " = " << render_const_value(type, value); + f_consts_ << endl << endl; +} + +/** + * Prints the value of a constant with the given type. Note that type checking + * is NOT performed in this function as it is always run beforehand using the + * validate_types method in main.cc + */ +string t_rb_generator::render_const_value(t_type* type, t_const_value* value) { + type = get_true_type(type); + std::ostringstream out; + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + out << "%q\"" << get_escaped_string(value) << '"'; + break; + case t_base_type::TYPE_BOOL: + out << (value->get_integer() > 0 ? "true" : "false"); + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + out << value->get_integer(); + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() == t_const_value::CV_INTEGER) { + out << value->get_integer(); + } else { + out << value->get_double(); + } + break; + default: + throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + indent(out) << value->get_integer(); + } else if (type->is_struct() || type->is_xception()) { + out << type->get_name() << ".new({" << endl; + indent_up(); + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + out << indent(); + out << render_const_value(g_type_string, v_iter->first); + out << " => "; + out << render_const_value(field_type, v_iter->second); + out << "," << endl; + } + indent_down(); + indent(out) << "})"; + } else if (type->is_map()) { + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + out << "{" << endl; + indent_up(); + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + out << indent(); + out << render_const_value(ktype, v_iter->first); + out << " => "; + out << render_const_value(vtype, v_iter->second); + out << "," << endl; + } + indent_down(); + indent(out) << "}"; + } else if (type->is_list() || type->is_set()) { + t_type* etype; + if (type->is_list()) { + etype = ((t_list*)type)->get_elem_type(); + } else { + etype = ((t_set*)type)->get_elem_type(); + } + if (type->is_set()) { + out << "Set.new(["; + } else { + out << "[" << endl; + } + indent_up(); + const vector& val = value->get_list(); + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + out << indent(); + out << render_const_value(etype, *v_iter); + out << "," << endl; + } + indent_down(); + if (type->is_set()) { + indent(out) << "])"; + } else { + indent(out) << "]"; + } + } else { + throw "CANNOT GENERATE CONSTANT FOR TYPE: " + type->get_name(); + } + return out.str(); +} + +/** + * Generates a ruby struct + */ +void t_rb_generator::generate_struct(t_struct* tstruct) { + generate_rb_struct(f_types_, tstruct, false); +} + +/** + * Generates a struct definition for a thrift exception. Basically the same + * as a struct but extends the Exception class. + * + * @param txception The struct definition + */ +void t_rb_generator::generate_xception(t_struct* txception) { + generate_rb_struct(f_types_, txception, true); +} + +/** + * Generates a ruby struct + */ +void t_rb_generator::generate_rb_struct(std::ofstream& out, t_struct* tstruct, bool is_exception = false) { + generate_rdoc(out, tstruct); + indent(out) << "class " << type_name(tstruct); + if (is_exception) { + out << " < ::Thrift::Exception"; + } + out << endl; + + indent_up(); + indent(out) << "include ::Thrift::Struct" << endl; + + if (is_exception) { + generate_rb_simple_exception_constructor(out, tstruct); + } + + generate_field_constants(out, tstruct); + generate_accessors(out, tstruct); + generate_field_defns(out, tstruct); + generate_rb_struct_required_validator(out, tstruct); + + indent_down(); + indent(out) << "end" << endl << endl; +} + +void t_rb_generator::generate_rb_simple_exception_constructor(std::ofstream& out, t_struct* tstruct) { + const vector& members = tstruct->get_members(); + + if (members.size() == 1) { + vector::const_iterator m_iter = members.begin(); + + if ((*m_iter)->get_type()->is_string()) { + string name = (*m_iter)->get_name(); + + indent(out) << "def initialize(message=nil)" << endl; + indent_up(); + indent(out) << "super()" << endl; + indent(out) << "self." << name << " = message" << endl; + indent_down(); + indent(out) << "end" << endl << endl; + + if (name != "message") { + indent(out) << "def message; " << name << " end" << endl << endl; + } + } + } +} + +void t_rb_generator::generate_field_constants(std::ofstream& out, t_struct* tstruct) { + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + std::string field_name = (*f_iter)->get_name(); + std::string cap_field_name = upcase_string(field_name); + + indent(out) << cap_field_name << " = " << (*f_iter)->get_key() << endl; + } + out << endl; +} + +void t_rb_generator::generate_accessors(std::ofstream& out, t_struct* tstruct) { + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + + if (members.size() > 0) { + indent(out) << "::Thrift::Struct.field_accessor self"; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + out << ", :" << (*m_iter)->get_name(); + } + out << endl; + } +} + +void t_rb_generator::generate_field_defns(std::ofstream& out, t_struct* tstruct) { + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + indent(out) << "FIELDS = {" << endl; + indent_up(); + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (f_iter != fields.begin()) { + out << "," << endl; + } + + // generate the field docstrings within the FIELDS constant. no real better place... + generate_rdoc(out, *f_iter); + + indent(out) << + upcase_string((*f_iter)->get_name()) << " => "; + + generate_field_data(out, (*f_iter)->get_type(), (*f_iter)->get_name(), (*f_iter)->get_value(), + (*f_iter)->get_req() == t_field::T_OPTIONAL); + } + indent_down(); + out << endl; + indent(out) << "}" << endl << endl; + + indent(out) << "def struct_fields; FIELDS; end" << endl << endl; + +} + +void t_rb_generator::generate_field_data(std::ofstream& out, t_type* field_type, + const std::string& field_name = "", t_const_value* field_value = NULL, bool optional = false) { + field_type = get_true_type(field_type); + + // Begin this field's defn + out << "{:type => " << type_to_enum(field_type); + + if (!field_name.empty()) { + out << ", :name => '" << field_name << "'"; + } + + if (field_value != NULL) { + out << ", :default => " << render_const_value(field_type, field_value); + } + + if (!field_type->is_base_type()) { + if (field_type->is_struct() || field_type->is_xception()) { + out << ", :class => " << full_type_name((t_struct*)field_type); + } else if (field_type->is_list()) { + out << ", :element => "; + generate_field_data(out, ((t_list*)field_type)->get_elem_type()); + } else if (field_type->is_map()) { + out << ", :key => "; + generate_field_data(out, ((t_map*)field_type)->get_key_type()); + out << ", :value => "; + generate_field_data(out, ((t_map*)field_type)->get_val_type()); + } else if (field_type->is_set()) { + out << ", :element => "; + generate_field_data(out, ((t_set*)field_type)->get_elem_type()); + } + } + + if(optional) { + out << ", :optional => true"; + } + + if (field_type->is_enum()) { + out << ", :enum_class => " << full_type_name(field_type); + } + + // End of this field's defn + out << "}"; +} + +void t_rb_generator::begin_namespace(std::ofstream& out, vector modules) { + for (vector::iterator m_iter = modules.begin(); m_iter != modules.end(); ++m_iter) { + indent(out) << "module " << *m_iter << endl; + indent_up(); + } +} + +void t_rb_generator::end_namespace(std::ofstream& out, vector modules) { + for (vector::reverse_iterator m_iter = modules.rbegin(); m_iter != modules.rend(); ++m_iter) { + indent_down(); + indent(out) << "end" << endl; + } +} + + +/** + * Generates a thrift service. + * + * @param tservice The service definition + */ +void t_rb_generator::generate_service(t_service* tservice) { + string f_service_name = get_out_dir()+underscore(service_name_)+".rb"; + f_service_.open(f_service_name.c_str()); + + f_service_ << + rb_autogen_comment() << endl << + "require 'thrift'" << endl; + + if (tservice->get_extends() != NULL) { + f_service_ << + "require '" << underscore(tservice->get_extends()->get_name()) << "'" << endl; + } + + f_service_ << + "require File.dirname(__FILE__) + '/" << underscore(program_name_) << "_types'" << endl << + endl; + + begin_namespace(f_service_, ruby_modules(tservice->get_program())); + + indent(f_service_) << "module " << capitalize(tservice->get_name()) << endl; + indent_up(); + + // Generate the three main parts of the service (well, two for now in PHP) + generate_service_client(tservice); + generate_service_server(tservice); + generate_service_helpers(tservice); + + indent_down(); + indent(f_service_) << "end" << endl << + endl; + + end_namespace(f_service_, ruby_modules(tservice->get_program())); + + // Close service file + f_service_.close(); +} + +/** + * Generates helper functions for a service. + * + * @param tservice The service to generate a header definition for + */ +void t_rb_generator::generate_service_helpers(t_service* tservice) { + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + indent(f_service_) << + "# HELPER FUNCTIONS AND STRUCTURES" << endl << endl; + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* ts = (*f_iter)->get_arglist(); + generate_rb_struct(f_service_, ts); + generate_rb_function_helpers(*f_iter); + } +} + +/** + * Generates a struct and helpers for a function. + * + * @param tfunction The function + */ +void t_rb_generator::generate_rb_function_helpers(t_function* tfunction) { + t_struct result(program_, tfunction->get_name() + "_result"); + t_field success(tfunction->get_returntype(), "success", 0); + if (!tfunction->get_returntype()->is_void()) { + result.append(&success); + } + + t_struct* xs = tfunction->get_xceptions(); + const vector& fields = xs->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + result.append(*f_iter); + } + generate_rb_struct(f_service_, &result); +} + +/** + * Generates a service client definition. + * + * @param tservice The service to generate a server for. + */ +void t_rb_generator::generate_service_client(t_service* tservice) { + string extends = ""; + string extends_client = ""; + if (tservice->get_extends() != NULL) { + extends = full_type_name(tservice->get_extends()); + extends_client = " < " + extends + "::Client "; + } + + indent(f_service_) << + "class Client" << extends_client << endl; + indent_up(); + + indent(f_service_) << + "include ::Thrift::Client" << endl << endl; + + // Generate client method implementations + vector functions = tservice->get_functions(); + vector::const_iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + t_struct* arg_struct = (*f_iter)->get_arglist(); + const vector& fields = arg_struct->get_members(); + vector::const_iterator fld_iter; + string funname = (*f_iter)->get_name(); + + // Open function + indent(f_service_) << + "def " << function_signature(*f_iter) << endl; + indent_up(); + indent(f_service_) << + "send_" << funname << "("; + + bool first = true; + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << (*fld_iter)->get_name(); + } + f_service_ << ")" << endl; + + if (!(*f_iter)->is_oneway()) { + f_service_ << indent(); + if (!(*f_iter)->get_returntype()->is_void()) { + f_service_ << "return "; + } + f_service_ << + "recv_" << funname << "()" << endl; + } + indent_down(); + indent(f_service_) << "end" << endl; + f_service_ << endl; + + indent(f_service_) << + "def send_" << function_signature(*f_iter) << endl; + indent_up(); + + std::string argsname = capitalize((*f_iter)->get_name() + "_args"); + + indent(f_service_) << "send_message('" << funname << "', " << argsname; + + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + f_service_ << ", :" << (*fld_iter)->get_name() << " => " << (*fld_iter)->get_name(); + } + + f_service_ << ")" << endl; + + indent_down(); + indent(f_service_) << "end" << endl; + + if (!(*f_iter)->is_oneway()) { + std::string resultname = capitalize((*f_iter)->get_name() + "_result"); + t_struct noargs(program_); + + t_function recv_function((*f_iter)->get_returntype(), + string("recv_") + (*f_iter)->get_name(), + &noargs); + // Open function + f_service_ << + endl << + indent() << "def " << function_signature(&recv_function) << endl; + indent_up(); + + // TODO(mcslee): Validate message reply here, seq ids etc. + + f_service_ << + indent() << "result = receive_message(" << resultname << ")" << endl; + + // Careful, only return _result if not a void function + if (!(*f_iter)->get_returntype()->is_void()) { + f_service_ << + indent() << "return result.success unless result.success.nil?" << endl; + } + + t_struct* xs = (*f_iter)->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + indent(f_service_) << + "raise result." << (*x_iter)->get_name() << + " unless result." << (*x_iter)->get_name() << ".nil?" << endl; + } + + // Careful, only return _result if not a void function + if ((*f_iter)->get_returntype()->is_void()) { + indent(f_service_) << + "return" << endl; + } else { + f_service_ << + indent() << "raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, '" << (*f_iter)->get_name() << " failed: unknown result')" << endl; + } + + // Close function + indent_down(); + indent(f_service_) << "end" << endl << endl; + } + } + + indent_down(); + indent(f_service_) << "end" << endl << endl; +} + +/** + * Generates a service server definition. + * + * @param tservice The service to generate a server for. + */ +void t_rb_generator::generate_service_server(t_service* tservice) { + // Generate the dispatch methods + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + string extends = ""; + string extends_processor = ""; + if (tservice->get_extends() != NULL) { + extends = full_type_name(tservice->get_extends()); + extends_processor = " < " + extends + "::Processor "; + } + + // Generate the header portion + indent(f_service_) << + "class Processor" << extends_processor << endl; + indent_up(); + + f_service_ << + indent() << "include ::Thrift::Processor" << endl << + endl; + + // Generate the process subfunctions + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + generate_process_function(tservice, *f_iter); + } + + indent_down(); + indent(f_service_) << "end" << endl << endl; +} + +/** + * Generates a process function definition. + * + * @param tfunction The function to write a dispatcher for + */ +void t_rb_generator::generate_process_function(t_service* tservice, + t_function* tfunction) { + // Open function + indent(f_service_) << + "def process_" << tfunction->get_name() << + "(seqid, iprot, oprot)" << endl; + indent_up(); + + string argsname = capitalize(tfunction->get_name()) + "_args"; + string resultname = capitalize(tfunction->get_name()) + "_result"; + + f_service_ << + indent() << "args = read_args(iprot, " << argsname << ")" << endl; + + t_struct* xs = tfunction->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + + // Declare result for non oneway function + if (!tfunction->is_oneway()) { + f_service_ << + indent() << "result = " << resultname << ".new()" << endl; + } + + // Try block for a function with exceptions + if (xceptions.size() > 0) { + f_service_ << + indent() << "begin" << endl; + indent_up(); + } + + // Generate the function call + t_struct* arg_struct = tfunction->get_arglist(); + const std::vector& fields = arg_struct->get_members(); + vector::const_iterator f_iter; + + f_service_ << indent(); + if (!tfunction->is_oneway() && !tfunction->get_returntype()->is_void()) { + f_service_ << "result.success = "; + } + f_service_ << + "@handler." << tfunction->get_name() << "("; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + f_service_ << ", "; + } + f_service_ << "args." << (*f_iter)->get_name(); + } + f_service_ << ")" << endl; + + if (!tfunction->is_oneway() && xceptions.size() > 0) { + indent_down(); + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + f_service_ << + indent() << "rescue " << full_type_name((*x_iter)->get_type()) << " => " << (*x_iter)->get_name() << endl; + if (!tfunction->is_oneway()) { + indent_up(); + f_service_ << + indent() << "result." << (*x_iter)->get_name() << " = " << (*x_iter)->get_name() << endl; + indent_down(); + } + } + indent(f_service_) << "end" << endl; + } + + // Shortcut out here for oneway functions + if (tfunction->is_oneway()) { + f_service_ << + indent() << "return" << endl; + indent_down(); + indent(f_service_) << "end" << endl << endl; + return; + } + + f_service_ << + indent() << "write_result(result, oprot, '" << tfunction->get_name() << "', seqid)" << endl; + + // Close function + indent_down(); + indent(f_service_) << "end" << endl << endl; +} + +/** + * Renders a function signature of the form 'type name(args)' + * + * @param tfunction Function definition + * @return String of rendered function definition + */ +string t_rb_generator::function_signature(t_function* tfunction, + string prefix) { + // TODO(mcslee): Nitpicky, no ',' if argument_list is empty + return + prefix + tfunction->get_name() + + "(" + argument_list(tfunction->get_arglist()) + ")"; +} + +/** + * Renders a field list + */ +string t_rb_generator::argument_list(t_struct* tstruct) { + string result = ""; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + result += ", "; + } + result += (*f_iter)->get_name(); + } + return result; +} + +string t_rb_generator::type_name(t_type* ttype) { + string prefix = ""; + + string name = ttype->get_name(); + if (ttype->is_struct() || ttype->is_xception() || ttype->is_enum()) { + name = capitalize(ttype->get_name()); + } + + return prefix + name; +} + +string t_rb_generator::full_type_name(t_type* ttype) { + string prefix = ""; + vector modules = ruby_modules(ttype->get_program()); + for (vector::iterator m_iter = modules.begin(); + m_iter != modules.end(); ++m_iter) { + prefix += *m_iter + "::"; + } + return prefix + type_name(ttype); +} + +/** + * Converts the parse type to a Ruby tyoe + */ +string t_rb_generator::type_to_enum(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + return "::Thrift::Types::STRING"; + case t_base_type::TYPE_BOOL: + return "::Thrift::Types::BOOL"; + case t_base_type::TYPE_BYTE: + return "::Thrift::Types::BYTE"; + case t_base_type::TYPE_I16: + return "::Thrift::Types::I16"; + case t_base_type::TYPE_I32: + return "::Thrift::Types::I32"; + case t_base_type::TYPE_I64: + return "::Thrift::Types::I64"; + case t_base_type::TYPE_DOUBLE: + return "::Thrift::Types::DOUBLE"; + } + } else if (type->is_enum()) { + return "::Thrift::Types::I32"; + } else if (type->is_struct() || type->is_xception()) { + return "::Thrift::Types::STRUCT"; + } else if (type->is_map()) { + return "::Thrift::Types::MAP"; + } else if (type->is_set()) { + return "::Thrift::Types::SET"; + } else if (type->is_list()) { + return "::Thrift::Types::LIST"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + + +void t_rb_generator::generate_rdoc(std::ofstream& out, t_doc* tdoc) { + if (tdoc->has_doc()) { + generate_docstring_comment(out, + "", "# ", tdoc->get_doc(), ""); + } +} + +void t_rb_generator::generate_rb_struct_required_validator(std::ofstream& out, + t_struct* tstruct) { + indent(out) << "def validate" << endl; + indent_up(); + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + t_field* field = (*f_iter); + if (field->get_req() == t_field::T_REQUIRED) { + indent(out) << "raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field " << field->get_name() << " is unset!')"; + if (field->get_type()->is_bool()) { + out << " if @" << field->get_name() << ".nil?"; + } else { + out << " unless @" << field->get_name(); + } + out << endl; + } + } + + // if field is an enum, check that its value is valid + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + t_field* field = (*f_iter); + + if (field->get_type()->is_enum()){ + indent(out) << "unless @" << field->get_name() << ".nil? || " << field->get_type()->get_name() << "::VALID_VALUES.include?(@" << field->get_name() << ")" << endl; + indent_up(); + indent(out) << "raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field " << field->get_name() << "!')" << endl; + indent_down(); + indent(out) << "end" << endl; + } + } + + indent_down(); + indent(out) << "end" << endl << endl; + +} + +THRIFT_REGISTER_GENERATOR(rb, "Ruby", ""); diff --git a/compiler/cpp/src/generate/t_st_generator.cc b/compiler/cpp/src/generate/t_st_generator.cc new file mode 100644 index 00000000..3600a3b8 --- /dev/null +++ b/compiler/cpp/src/generate/t_st_generator.cc @@ -0,0 +1,1071 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "platform.h" +#include "t_oop_generator.h" +using namespace std; + + +/** + * Smalltalk code generator. + * + */ +class t_st_generator : public t_oop_generator { + public: + t_st_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_oop_generator(program) + { + out_dir_base_ = "gen-st"; + } + + /** + * Init and close methods + */ + + void init_generator(); + void close_generator(); + + /** + * Program-level generation functions + */ + + void generate_typedef (t_typedef* ttypedef); + void generate_enum (t_enum* tenum); + void generate_const (t_const* tconst); + void generate_struct (t_struct* tstruct); + void generate_xception (t_struct* txception); + void generate_service (t_service* tservice); + void generate_class_side_definition (); + void generate_force_consts (); + + + std::string render_const_value(t_type* type, t_const_value* value); + + /** + * Struct generation code + */ + + void generate_st_struct (std::ofstream& out, t_struct* tstruct, bool is_exception); + void generate_accessors (std::ofstream& out, t_struct* tstruct); + + /** + * Service-level generation functions + */ + + void generate_service_client (t_service* tservice); + + void generate_send_method (t_function* tfunction); + void generate_recv_method (t_function* tfunction); + + std::string map_reader (t_map *tmap); + std::string list_reader (t_list *tlist); + std::string set_reader (t_set *tset); + std::string struct_reader (t_struct *tstruct, std::string clsName); + + std::string map_writer (t_map *tmap, std::string name); + std::string list_writer (t_list *tlist, std::string name); + std::string set_writer (t_set *tset, std::string name); + std::string struct_writer (t_struct *tstruct, std::string fname); + + std::string write_val (t_type *t, std::string fname); + std::string read_val (t_type *t); + + /** + * Helper rendering functions + */ + + std::string st_autogen_comment(); + + void st_class_def(std::ofstream &out, std::string name); + void st_method(std::ofstream &out, std::string cls, std::string name); + void st_method(std::ofstream &out, std::string cls, std::string name, std::string category); + void st_close_method(std::ofstream &out); + void st_class_method(std::ofstream &out, std::string cls, std::string name); + void st_class_method(std::ofstream &out, std::string cls, std::string name, std::string category); + void st_setter(std::ofstream &out, std::string cls, std::string name, std::string type); + void st_getter(std::ofstream &out, std::string cls, std::string name); + void st_accessors(std::ofstream &out, std::string cls, std::string name, std::string type); + + std::string class_name(); + std::string client_class_name(); + std::string prefix(std::string name); + std::string declare_field(t_field* tfield); + std::string sanitize(std::string s); + std::string type_name(t_type* ttype); + + std::string function_signature(t_function* tfunction); + std::string argument_list(t_struct* tstruct); + std::string function_types_comment(t_function* fn); + + std::string type_to_enum(t_type* ttype); + std::string a_type(t_type* type); + bool is_vowel(char c); + std::string temp_name(); + std::string generated_category(); + + private: + + /** + * File streams + */ + int temporary_var; + std::ofstream f_; + +}; + + +/** + * Prepares for file generation by opening up the necessary file output + * streams. + * + * @param tprogram The program to generate + */ +void t_st_generator::init_generator() { + // Make output directory + MKDIR(get_out_dir().c_str()); + + temporary_var = 0; + + // Make output file + string f_name = get_out_dir()+"/"+program_name_+".st"; + f_.open(f_name.c_str()); + + // Print header + f_ << st_autogen_comment() << endl; + + st_class_def(f_, program_name_); + generate_class_side_definition(); + + //Generate enums + vector enums = program_->get_enums(); + vector::iterator en_iter; + for (en_iter = enums.begin(); en_iter != enums.end(); ++en_iter) { + generate_enum(*en_iter); + } +} + +string t_st_generator::class_name() { + return capitalize(program_name_); +} + +string t_st_generator::prefix(string class_name) { + string prefix = program_->get_namespace("smalltalk.prefix"); + string name = capitalize(class_name); + name = prefix.empty() ? name : (prefix + name); + return name; +} + +string t_st_generator::client_class_name() { + return capitalize(service_name_) + "Client"; +} + +/** + * Autogen'd comment + */ +string t_st_generator::st_autogen_comment() { + return + std::string("'") + + "Autogenerated by Thrift\n" + + "\n" + + "DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n" + + "'!\n"; +} + +void t_st_generator::generate_force_consts() { + f_ << prefix(class_name()) << " enums keysAndValuesDo: [:k :v | " << + prefix(class_name()) << " enums at: k put: v value].!" << endl; + + f_ << prefix(class_name()) << " constants keysAndValuesDo: [:k :v | " << + prefix(class_name()) << " constants at: k put: v value].!" << endl; + +} + +void t_st_generator::close_generator() { + generate_force_consts(); + f_.close(); +} + +string t_st_generator::generated_category() { + string cat = program_->get_namespace("smalltalk.category"); + // For compatibility with the Thrift grammar, the category must + // be punctuated by dots. Replaces them with dashes here. + for (string::iterator iter = cat.begin(); iter != cat.end(); ++iter) { + if (*iter == '.') { + *iter = '-'; + } + } + return cat.size() ? cat : "Generated-" + class_name(); +} + +/** + * Generates a typedef. This is not done in Smalltalk, types are all implicit. + * + * @param ttypedef The type definition + */ +void t_st_generator::generate_typedef(t_typedef* ttypedef) {} + +void t_st_generator::st_class_def(std::ofstream &out, string name) { + out << "Object subclass: #" << prefix(name) << endl; + indent_up(); + out << indent() << "instanceVariableNames: ''" << endl << + indent() << "classVariableNames: ''" << endl << + indent() << "poolDictionaries: ''" << endl << + indent() << "category: '" << generated_category() << "'!" << endl << endl; +} + +void t_st_generator::st_method(std::ofstream &out, string cls, string name) { + st_method(out, cls, name, "as yet uncategorized"); +} + +void t_st_generator::st_class_method(std::ofstream &out, string cls, string name) { + st_method(out, cls + " class", name); +} + +void t_st_generator::st_class_method(std::ofstream &out, string cls, string name, string category) { + st_method(out, cls, name, category); +} + +void t_st_generator::st_method(std::ofstream &out, string cls, string name, string category) { + char timestr[50]; + time_t rawtime; + struct tm *tinfo; + + time(&rawtime); + tinfo = localtime(&rawtime); + strftime(timestr, 50, "%m/%d/%Y %H:%M", tinfo); + + out << "!" << prefix(cls) << + " methodsFor: '"+category+"' stamp: 'thrift " << timestr << "'!\n" << + name << endl; + + indent_up(); + out << indent(); +} + +void t_st_generator::st_close_method(std::ofstream &out) { + out << "! !" << endl << endl; + indent_down(); +} + +void t_st_generator::st_setter(std::ofstream &out, string cls, string name, string type = "anObject") { + st_method(out, cls, name + ": " + type); + out << name << " := " + type; + st_close_method(out); +} + +void t_st_generator::st_getter(std::ofstream &out, string cls, string name) { + st_method(out, cls, name + ""); + out << "^ " << name; + st_close_method(out); +} + +void t_st_generator::st_accessors(std::ofstream &out, string cls, string name, string type = "anObject") { + st_setter(out, cls, name, type); + st_getter(out, cls, name); +} + +void t_st_generator::generate_class_side_definition() { + f_ << prefix(class_name()) << " class" << endl << + "\tinstanceVariableNames: 'constants enums'!" << endl << endl; + + st_accessors(f_, class_name() + " class", "enums"); + st_accessors(f_, class_name() + " class", "constants"); + + f_ << prefix(class_name()) << " enums: Dictionary new!" << endl; + f_ << prefix(class_name()) << " constants: Dictionary new!" << endl; + + f_ << endl; +} + +/** + * Generates code for an enumerated type. Done using a class to scope + * the values. + * + * @param tenum The enumeration + */ +void t_st_generator::generate_enum(t_enum* tenum) { + string cls_name = program_name_ + capitalize(tenum->get_name()); + + f_ << prefix(class_name()) << " enums at: '" << tenum->get_name() << "' put: [" << + "(Dictionary new " << endl; + + vector constants = tenum->get_constants(); + vector::iterator c_iter; + int value = -1; + for (c_iter = constants.begin(); c_iter != constants.end(); ++c_iter) { + if ((*c_iter)->has_value()) { + value = (*c_iter)->get_value(); + } else { + ++value; + } + + f_ << "\tat: '" << (*c_iter)->get_name() << "' put: " << value << ";" << endl; + } + + f_ << "\tyourself)]!" << endl << endl; +} + +/** + * Generate a constant value + */ +void t_st_generator::generate_const(t_const* tconst) { + t_type* type = tconst->get_type(); + string name = tconst->get_name(); + t_const_value* value = tconst->get_value(); + + f_ << prefix(class_name()) << " constants at: '" << name << "' put: [" << + render_const_value(type, value) << "]!" << endl << endl; +} + +/** + * Prints the value of a constant with the given type. Note that type checking + * is NOT performed in this function as it is always run beforehand using the + * validate_types method in main.cc + */ +string t_st_generator::render_const_value(t_type* type, t_const_value* value) { + type = get_true_type(type); + std::ostringstream out; + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + out << '"' << get_escaped_string(value) << '"'; + break; + case t_base_type::TYPE_BOOL: + out << (value->get_integer() > 0 ? "true" : "false"); + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + out << value->get_integer(); + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() == t_const_value::CV_INTEGER) { + out << value->get_integer(); + } else { + out << value->get_double(); + } + break; + default: + throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase); + } + } else if (type->is_enum()) { + indent(out) << value->get_integer(); + } else if (type->is_struct() || type->is_xception()) { + out << "(" << capitalize(type->get_name()) << " new " << endl; + indent_up(); + + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + const map& val = value->get_map(); + map::const_iterator v_iter; + + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + + out << indent() << v_iter->first->get_string() << ": " << + render_const_value(field_type, v_iter->second) << ";" << endl; + } + out << indent() << "yourself)"; + + indent_down(); + } else if (type->is_map()) { + t_type* ktype = ((t_map*)type)->get_key_type(); + t_type* vtype = ((t_map*)type)->get_val_type(); + out << "(Dictionary new" << endl; + indent_up(); + indent_up(); + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + out << indent() << indent(); + out << "at: " << render_const_value(ktype, v_iter->first); + out << " put: "; + out << render_const_value(vtype, v_iter->second); + out << ";" << endl; + } + out << indent() << indent() << "yourself)"; + indent_down(); + indent_down(); + } else if (type->is_list() || type->is_set()) { + t_type* etype; + if (type->is_list()) { + etype = ((t_list*)type)->get_elem_type(); + } else { + etype = ((t_set*)type)->get_elem_type(); + } + if (type->is_set()) { + out << "(Set new" << endl; + } else { + out << "(OrderedCollection new" << endl; + } + indent_up(); + indent_up(); + const vector& val = value->get_list(); + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + out << indent() << indent(); + out << "add: " << render_const_value(etype, *v_iter); + out << ";" << endl; + } + out << indent() << indent() << "yourself)"; + indent_down(); + indent_down(); + } else { + throw "CANNOT GENERATE CONSTANT FOR TYPE: " + type->get_name(); + } + return out.str(); +} + +/** + * Generates a Smalltalk struct + */ +void t_st_generator::generate_struct(t_struct* tstruct) { + generate_st_struct(f_, tstruct, false); +} + +/** + * Generates a struct definition for a thrift exception. Basically the same + * as a struct but extends the Exception class. + * + * @param txception The struct definition + */ +void t_st_generator::generate_xception(t_struct* txception) { + generate_st_struct(f_, txception, true); +} + +/** + * Generates a smalltalk class to represent a struct + */ +void t_st_generator::generate_st_struct(std::ofstream& out, t_struct* tstruct, bool is_exception = false) { + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + + if (is_exception) + out << "Error"; + else + out << "Object"; + + out << " subclass: #" << prefix(type_name(tstruct)) << endl << + "\tinstanceVariableNames: '"; + + if (members.size() > 0) { + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + if (m_iter != members.begin()) out << " "; + out << sanitize((*m_iter)->get_name()); + } + } + + out << "'\n" << + "\tclassVariableNames: ''\n" << + "\tpoolDictionaries: ''\n" << + "\tcategory: '" << generated_category() << "'!\n\n"; + + generate_accessors(out, tstruct); +} + +bool t_st_generator::is_vowel(char c) { + switch(tolower(c)) { + case 'a': case 'e': case 'i': case 'o': case 'u': + return true; + } + return false; +} + +string t_st_generator::a_type(t_type* type) { + string prefix; + + if (is_vowel(type_name(type)[0])) + prefix = "an"; + else + prefix = "a"; + + return prefix + capitalize(type_name(type)); +} + +void t_st_generator::generate_accessors(std::ofstream& out, t_struct* tstruct) { + const vector& members = tstruct->get_members(); + vector::const_iterator m_iter; + string type; + string prefix; + + if (members.size() > 0) { + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + st_accessors(out, + capitalize(type_name(tstruct)), + sanitize((*m_iter)->get_name()), + a_type((*m_iter)->get_type())); + } + out << endl; + } +} + +/** + * Generates a thrift service. + * + * @param tservice The service definition + */ +void t_st_generator::generate_service(t_service* tservice) { + generate_service_client(tservice); + // generate_service_server(tservice); +} + +string t_st_generator::temp_name() { + std::ostringstream out; + out << "temp" << temporary_var++; + return out.str(); +} + +string t_st_generator::map_writer(t_map *tmap, string fname) { + std::ostringstream out; + string key = temp_name(); + string val = temp_name(); + + out << "[oprot writeMapBegin: (TMap new keyType: " << type_to_enum(tmap->get_key_type()) << + "; valueType: " << type_to_enum(tmap->get_val_type()) << "; size: " << fname << " size)." << endl; + indent_up(); + + out << indent() << fname << " keysAndValuesDo: [:" << key << " :" << val << " |" << endl; + indent_up(); + + out << indent() << write_val(tmap->get_key_type(), key) << "." << endl << + indent() << write_val(tmap->get_val_type(), val); + indent_down(); + + out << "]." << endl << + indent() << "oprot writeMapEnd] value"; + indent_down(); + + return out.str(); +} + +string t_st_generator::map_reader(t_map *tmap) { + std::ostringstream out; + string desc = temp_name(); + string val = temp_name(); + + out << "[|" << desc << " " << val << "| " << endl; + indent_up(); + + out << indent() << desc << " := iprot readMapBegin." << endl << + indent() << val << " := Dictionary new." << endl << + indent() << desc << " size timesRepeat: [" << endl; + + indent_up(); + out << indent() << val << " at: " << read_val(tmap->get_key_type()) << + " put: " << read_val(tmap->get_val_type()); + indent_down(); + + out << "]." << endl << + indent() << "iprot readMapEnd." << endl << + indent() << val << "] value"; + indent_down(); + + return out.str(); +} + +string t_st_generator::list_writer(t_list *tlist, string fname) { + std::ostringstream out; + string val = temp_name(); + + out << "[oprot writeListBegin: (TList new elemType: " << + type_to_enum(tlist->get_elem_type()) << "; size: " << fname << " size)." << endl; + indent_up(); + + out << indent() << fname << " do: [:" << val << "|" << endl; + indent_up(); + + out << indent() << write_val(tlist->get_elem_type(), val) << endl; + indent_down(); + + out << "]." << endl << + indent() << "oprot writeListEnd] value"; + indent_down(); + + return out.str(); +} + +string t_st_generator::list_reader(t_list *tlist) { + std::ostringstream out; + string desc = temp_name(); + string val = temp_name(); + + out << "[|" << desc << " " << val << "| " << desc << " := iprot readListBegin." << endl; + indent_up(); + + out << indent() << val << " := OrderedCollection new." << endl << + indent() << desc << " size timesRepeat: [" << endl; + + indent_up(); + out << indent() << val << " add: " << read_val(tlist->get_elem_type()); + indent_down(); + + out << "]." << endl << + indent() << "iprot readListEnd." << endl << + indent() << val << "] value"; + indent_down(); + + return out.str(); +} + +string t_st_generator::set_writer(t_set *tset, string fname) { + std::ostringstream out; + string val = temp_name(); + + out << "[oprot writeSetBegin: (TSet new elemType: " << type_to_enum(tset->get_elem_type()) << + "; size: " << fname << " size)." << endl; + indent_up(); + + out << indent() << fname << " do: [:" << val << "|" << endl; + indent_up(); + + out << indent() << write_val(tset->get_elem_type(), val) << endl; + indent_down(); + + out << "]." << endl << + indent() << "oprot writeSetEnd] value"; + indent_down(); + + return out.str(); +} + +string t_st_generator::set_reader(t_set *tset) { + std::ostringstream out; + string desc = temp_name(); + string val = temp_name(); + + out << "[|" << desc << " " << val << "| " << desc << " := iprot readSetBegin." << endl; + indent_up(); + + out << indent() << val << " := Set new." << endl << + indent() << desc << " size timesRepeat: [" << endl; + + indent_up(); + out << indent() << val << " add: " << read_val(tset->get_elem_type()); + indent_down(); + + out << "]." << endl << + indent() << "iprot readSetEnd." << endl << + indent() << val << "] value"; + indent_down(); + + return out.str(); +} + +string t_st_generator::struct_writer(t_struct *tstruct, string sname) { + std::ostringstream out; + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator fld_iter; + + out << "[oprot writeStructBegin: " << + "(TStruct new name: '" + tstruct->get_name() +"')." << endl; + indent_up(); + + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + bool optional = (*fld_iter)->get_req() == t_field::T_OPTIONAL; + string fname = (*fld_iter)->get_name(); + string accessor = sname + " " + sanitize(fname); + + if (optional) { + out << indent() << accessor << " ifNotNil: [" << endl; + indent_up(); + } + + out << indent() << "oprot writeFieldBegin: (TField new name: '" << fname << + "'; type: " << type_to_enum((*fld_iter)->get_type()) << + "; id: " << (*fld_iter)->get_key() << ")." << endl; + + out << indent() << write_val((*fld_iter)->get_type(), accessor) << "." << endl << + indent() << "oprot writeFieldEnd"; + + if (optional) { + out << "]"; + indent_down(); + } + + out << "." << endl; + } + + out << indent() << "oprot writeFieldStop; writeStructEnd] value"; + indent_down(); + + return out.str(); +} + +string t_st_generator::struct_reader(t_struct *tstruct, string clsName = "") { + std::ostringstream out; + const vector& fields = tstruct->get_members(); + vector::const_iterator fld_iter; + string val = temp_name(); + string desc = temp_name(); + string found = temp_name(); + + if (clsName.size() == 0) { + clsName = tstruct->get_name(); + } + + out << "[|" << desc << " " << val << "|" << endl; + indent_up(); + + //This is nasty, but without it we'll break things by prefixing TResult. + string name = ((capitalize(clsName) == "TResult") ? capitalize(clsName) : prefix(clsName)); + out << indent() << val << " := " << name << " new." << endl; + + out << indent() << "iprot readStructBegin." << endl << + indent() << "[" << desc << " := iprot readFieldBegin." << endl << + indent() << desc << " type = TType stop] whileFalse: [|" << found << "|" << endl; + indent_up(); + + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + out << indent() << desc << " id = " << (*fld_iter)->get_key() << + " ifTrue: [" << endl; + indent_up(); + + out << indent() << found << " := true." << endl << + indent() << val << " " << sanitize((*fld_iter)->get_name()) << ": " << + read_val((*fld_iter)->get_type()); + indent_down(); + + out << "]." << endl; + } + + out << indent() << found << " ifNil: [iprot skip: " << desc << " type]]." << endl; + indent_down(); + + out << indent() << "oprot readStructEnd." << endl << + indent() << val << "] value"; + indent_down(); + + return out.str(); +} + +string t_st_generator::write_val(t_type *t, string fname) { + t = get_true_type(t); + + if (t->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*) t)->get_base(); + switch(tbase) { + case t_base_type::TYPE_DOUBLE: + return "iprot writeDouble: " + fname + " asFloat"; + break; + case t_base_type::TYPE_BYTE: + case t_base_type::TYPE_I16: + case t_base_type::TYPE_I32: + case t_base_type::TYPE_I64: + return "iprot write" + capitalize(type_name(t)) + ": " + fname + " asInteger"; + default: + return "iprot write" + capitalize(type_name(t)) + ": " + fname; + } + } else if (t->is_map()) { + return map_writer((t_map*) t, fname); + } else if (t->is_struct() || t->is_xception()) { + return struct_writer((t_struct*) t, fname); + } else if (t->is_list()) { + return list_writer((t_list*) t, fname); + } else if (t->is_set()) { + return set_writer((t_set*) t, fname); + } else if (t->is_enum()) { + return "iprot writeI32: " + fname; + } else { + throw "Sorry, I don't know how to write this: " + type_name(t); + } +} + +string t_st_generator::read_val(t_type *t) { + t = get_true_type(t); + + if (t->is_base_type()) { + return "iprot read" + capitalize(type_name(t)); + } else if (t->is_map()) { + return map_reader((t_map*) t); + } else if (t->is_struct() || t->is_xception()) { + return struct_reader((t_struct*) t); + } else if (t->is_list()) { + return list_reader((t_list*) t); + } else if (t->is_set()) { + return set_reader((t_set*) t); + } else if (t->is_enum()) { + return "iprot readI32"; + } else { + throw "Sorry, I don't know how to read this: " + type_name(t); + } +} + +void t_st_generator::generate_send_method(t_function* function) { + string funname = function->get_name(); + string signature = function_signature(function); + t_struct* arg_struct = function->get_arglist(); + const vector& fields = arg_struct->get_members(); + vector::const_iterator fld_iter; + + st_method(f_, client_class_name(), "send" + capitalize(signature)); + f_ << "oprot writeMessageBegin:" << endl; + indent_up(); + + f_ << indent() << "(TCallMessage new" << endl; + indent_up(); + + f_ << indent() << "name: '" << funname << "'; " << endl << + indent() << "seqid: self nextSeqid)." << endl; + indent_down(); + indent_down(); + + f_ << indent() << "oprot writeStructBegin: " << + "(TStruct new name: '" + capitalize(function->get_name()) + "_args')." << endl; + + for (fld_iter = fields.begin(); fld_iter != fields.end(); ++fld_iter) { + string fname = (*fld_iter)->get_name(); + + f_ << indent() << "oprot writeFieldBegin: (TField new name: '" << fname << + "'; type: " << type_to_enum((*fld_iter)->get_type()) << + "; id: " << (*fld_iter)->get_key() << ")." << endl; + + f_ << indent() << write_val((*fld_iter)->get_type(), fname) << "." << endl << + indent() << "oprot writeFieldEnd." << endl; + } + + f_ << indent() << "oprot writeFieldStop; writeStructEnd; writeMessageEnd." << endl; + f_ << indent() << "oprot transport flush"; + + st_close_method(f_); +} + +// We only support receiving TResult structures (so this won't work on the server side) +void t_st_generator::generate_recv_method(t_function* function) { + string funname = function->get_name(); + string signature = function_signature(function); + + t_struct result(program_, "TResult"); + t_field success(function->get_returntype(), "success", 0); + result.append(&success); + + t_struct* xs = function->get_xceptions(); + const vector& fields = xs->get_members(); + vector::const_iterator f_iter; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + // duplicate the field, but call it "exception"... we don't need a dynamic name + t_field *exception = new t_field((*f_iter)->get_type(), "exception", (*f_iter)->get_key()); + result.append(exception); + } + + st_method(f_, client_class_name(), "recv" + capitalize(funname)); + f_ << "| f msg res | " << endl << + indent() << "msg := oprot readMessageBegin." << endl << + indent() << "self validateRemoteMessage: msg." << endl << + indent() << "res := " << struct_reader(&result) << "." << endl << + indent() << "oprot readMessageEnd." << endl << + indent() << "oprot transport flush." << endl << + indent() << "res exception ifNotNil: [res exception signal]." << endl << + indent() << "^ res"; + st_close_method(f_); +} + +string t_st_generator::function_types_comment(t_function* fn) { + std::ostringstream out; + const vector& fields = fn->get_arglist()->get_members(); + vector::const_iterator f_iter; + + out << "\""; + + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + out << (*f_iter)->get_name() << ": " << type_name((*f_iter)->get_type()); + if ((f_iter + 1) != fields.end()) { + out << ", "; + } + } + + out << "\""; + + return out.str(); +} + +/** + * Generates a service client definition. + * + * @param tservice The service to generate a server for. + */ +void t_st_generator::generate_service_client(t_service* tservice) { + string extends = ""; + string extends_client = "TClient"; + vector functions = tservice->get_functions(); + vector::iterator f_iter; + + if (tservice->get_extends() != NULL) { + extends = type_name(tservice->get_extends()); + extends_client = extends + "Client"; + } + + f_ << extends_client << " subclass: #" << prefix(client_class_name()) << endl << + "\tinstanceVariableNames: ''\n" << + "\tclassVariableNames: ''\n" << + "\tpoolDictionaries: ''\n" << + "\tcategory: '" << generated_category() << "'!\n\n"; + + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string funname = (*f_iter)->get_name(); + string signature = function_signature(*f_iter); + + st_method(f_, client_class_name(), signature); + f_ << function_types_comment(*f_iter) << endl << + indent() << "self send" << capitalize(signature) << "." << endl; + + if (!(*f_iter)->is_oneway()) { + f_ << indent() << "^ self recv" << capitalize(funname) << " success " << endl; + } + + st_close_method(f_); + + generate_send_method(*f_iter); + if (!(*f_iter)->is_oneway()) { + generate_recv_method(*f_iter); + } + } +} + +string t_st_generator::sanitize(string s) { + std::ostringstream out; + bool underscore = false; + + for (unsigned int i = 0; i < s.size(); i++) { + if (s[i] == '_') { + underscore = true; + continue; + } + if (underscore) { + out << (char) toupper(s[i]); + underscore = false; + continue; + } + out << s[i]; + } + + return out.str(); +} + +/** + * Renders a function signature of the form 'type name(args)' + * + * @param tfunction Function definition + * @return String of rendered function definition + */ +string t_st_generator::function_signature(t_function* tfunction) { + return tfunction->get_name() + capitalize(argument_list(tfunction->get_arglist())); +} + +/** + * Renders a field list + */ +string t_st_generator::argument_list(t_struct* tstruct) { + string result = ""; + + const vector& fields = tstruct->get_members(); + vector::const_iterator f_iter; + bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if (first) { + first = false; + } else { + result += " "; + } + result += (*f_iter)->get_name() + ": " + (*f_iter)->get_name(); + } + return result; +} + +string t_st_generator::type_name(t_type* ttype) { + string prefix = ""; + t_program* program = ttype->get_program(); + if (program != NULL && program != program_) { + if (!ttype->is_service()) { + prefix = program->get_name() + "_types."; + } + } + + string name = ttype->get_name(); + if (ttype->is_struct() || ttype->is_xception()) { + name = capitalize(ttype->get_name()); + } + + return prefix + name; +} + +/* Convert t_type to Smalltalk type code */ +string t_st_generator::type_to_enum(t_type* type) { + type = get_true_type(type); + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_VOID: + throw "NO T_VOID CONSTRUCT"; + case t_base_type::TYPE_STRING: + return "TType string"; + case t_base_type::TYPE_BOOL: + return "TType bool"; + case t_base_type::TYPE_BYTE: + return "TType byte"; + case t_base_type::TYPE_I16: + return "TType i16"; + case t_base_type::TYPE_I32: + return "TType i32"; + case t_base_type::TYPE_I64: + return "TType i64"; + case t_base_type::TYPE_DOUBLE: + return "TType double"; + } + } else if (type->is_enum()) { + return "TType i32"; + } else if (type->is_struct() || type->is_xception()) { + return "TType struct"; + } else if (type->is_map()) { + return "TType map"; + } else if (type->is_set()) { + return "TType set"; + } else if (type->is_list()) { + return "TType list"; + } + + throw "INVALID TYPE IN type_to_enum: " + type->get_name(); +} + + +THRIFT_REGISTER_GENERATOR(st, "Smalltalk", ""); diff --git a/compiler/cpp/src/generate/t_xsd_generator.cc b/compiler/cpp/src/generate/t_xsd_generator.cc new file mode 100644 index 00000000..729a91ae --- /dev/null +++ b/compiler/cpp/src/generate/t_xsd_generator.cc @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include +#include +#include +#include "t_generator.h" +#include "platform.h" +using namespace std; + + +/** + * XSD generator, creates an XSD for the base types etc. + * + */ +class t_xsd_generator : public t_generator { + public: + t_xsd_generator( + t_program* program, + const std::map& parsed_options, + const std::string& option_string) + : t_generator(program) + { + out_dir_base_ = "gen-xsd"; + } + + virtual ~t_xsd_generator() {} + + /** + * Init and close methods + */ + + void init_generator(); + void close_generator(); + + /** + * Program-level generation functions + */ + + void generate_typedef(t_typedef* ttypedef); + void generate_enum(t_enum* tenum) {} + + void generate_service(t_service* tservice); + void generate_struct(t_struct* tstruct); + + private: + + void generate_element(std::ostream& out, std::string name, t_type* ttype, t_struct* attrs=NULL, bool optional=false, bool nillable=false, bool list_element=false); + + std::string ns(std::string in, std::string ns) { + return ns + ":" + in; + } + + std::string xsd(std::string in) { + return ns(in, "xsd"); + } + + std::string type_name(t_type* ttype); + std::string base_type_name(t_base_type::t_base tbase); + + /** + * Output xsd/php file + */ + std::ofstream f_xsd_; + std::ofstream f_php_; + + /** + * Output string stream + */ + std::ostringstream s_xsd_types_; + +}; + + +void t_xsd_generator::init_generator() { + // Make output directory + MKDIR(get_out_dir().c_str()); + + // Make output file + string f_php_name = get_out_dir()+program_->get_name()+"_xsd.php"; + f_php_.open(f_php_name.c_str()); + + f_php_ << + "" << endl; + f_php_.close(); +} + +void t_xsd_generator::generate_typedef(t_typedef* ttypedef) { + indent(s_xsd_types_) << + "get_name() << "\">" << endl; + indent_up(); + if (ttypedef->get_type()->is_string() && ((t_base_type*)ttypedef->get_type())->is_string_enum()) { + indent(s_xsd_types_) << + "get_type()) << "\">" << endl; + indent_up(); + const vector& values = ((t_base_type*)ttypedef->get_type())->get_string_enum_vals(); + vector::const_iterator v_iter; + for (v_iter = values.begin(); v_iter != values.end(); ++v_iter) { + indent(s_xsd_types_) << + "" << endl; + } + indent_down(); + indent(s_xsd_types_) << + "" << endl; + } else { + indent(s_xsd_types_) << + "get_type()) << "\" />" << endl; + } + indent_down(); + indent(s_xsd_types_) << + "" << endl << endl; +} + +void t_xsd_generator::generate_struct(t_struct* tstruct) { + vector::const_iterator m_iter; + const vector& members = tstruct->get_members(); + bool xsd_all = tstruct->get_xsd_all(); + + indent(s_xsd_types_) << "get_name() << "\">" << endl; + indent_up(); + if (xsd_all) { + indent(s_xsd_types_) << "" << endl; + } else { + indent(s_xsd_types_) << "" << endl; + } + indent_up(); + + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + generate_element(s_xsd_types_, (*m_iter)->get_name(), (*m_iter)->get_type(), (*m_iter)->get_xsd_attrs(), (*m_iter)->get_xsd_optional() || xsd_all, (*m_iter)->get_xsd_nillable()); + } + + indent_down(); + if (xsd_all) { + indent(s_xsd_types_) << "" << endl; + } else { + indent(s_xsd_types_) << "" << endl; + } + indent_down(); + indent(s_xsd_types_) << + "" << endl << + endl; +} + +void t_xsd_generator::generate_element(ostream& out, + string name, + t_type* ttype, + t_struct* attrs, + bool optional, + bool nillable, + bool list_element) { + string sminOccurs = (optional || list_element) ? " minOccurs=\"0\"" : ""; + string smaxOccurs = list_element ? " maxOccurs=\"unbounded\"" : ""; + string soptional = sminOccurs + smaxOccurs; + string snillable = nillable ? " nillable=\"true\"" : ""; + + if (ttype->is_void() || ttype->is_list()) { + indent(out) << + "" << endl; + indent_up(); + if (attrs == NULL && ttype->is_void()) { + indent(out) << + "" << endl; + } else { + indent(out) << + "" << endl; + indent_up(); + if (ttype->is_list()) { + indent(out) << "" << endl; + indent_up(); + string subname; + t_type* subtype = ((t_list*)ttype)->get_elem_type(); + if (subtype->is_base_type() || subtype->is_container()) { + subname = name + "_elt"; + } else { + subname = type_name(subtype); + } + f_php_ << "$GLOBALS['" << program_->get_name() << "_xsd_elt_" << name << "'] = '" << subname << "';" << endl; + generate_element(out, subname, subtype, NULL, false, false, true); + indent_down(); + indent(out) << "" << endl; + indent(out) << "" << endl; + } + if (attrs != NULL) { + const vector& members = attrs->get_members(); + vector::const_iterator a_iter; + for (a_iter = members.begin(); a_iter != members.end(); ++a_iter) { + indent(out) << "get_name() << "\" type=\"" << type_name((*a_iter)->get_type()) << "\" />" << endl; + } + } + indent_down(); + indent(out) << + "" << endl; + } + indent_down(); + indent(out) << + "" << endl; + } else { + if (attrs == NULL) { + indent(out) << + "" << endl; + } else { + // Wow, all this work for a SIMPLE TYPE with attributes?!?!?! + indent(out) << "" << endl; + indent_up(); + indent(out) << "" << endl; + indent_up(); + indent(out) << "" << endl; + indent_up(); + indent(out) << "" << endl; + indent_up(); + const vector& members = attrs->get_members(); + vector::const_iterator a_iter; + for (a_iter = members.begin(); a_iter != members.end(); ++a_iter) { + indent(out) << "get_name() << "\" type=\"" << type_name((*a_iter)->get_type()) << "\" />" << endl; + } + indent_down(); + indent(out) << "" << endl; + indent_down(); + indent(out) << "" << endl; + indent_down(); + indent(out) << "" << endl; + indent_down(); + indent(out) << "" << endl; + } + } +} + +void t_xsd_generator::generate_service(t_service* tservice) { + // Make output file + string f_xsd_name = get_out_dir()+tservice->get_name()+".xsd"; + f_xsd_.open(f_xsd_name.c_str()); + + string ns = program_->get_namespace("xsd"); + if (ns.size() > 0) { + ns = " targetNamespace=\"" + ns + "\" xmlns=\"" + ns + "\" " + + "elementFormDefault=\"qualified\""; + } + + // Print the XSD header + f_xsd_ << + "" << endl << + "" << endl << + endl << + "" << endl << + endl; + + // Print out the type definitions + indent(f_xsd_) << s_xsd_types_.str(); + + // Keep a list of all the possible exceptions that might get thrown + map all_xceptions; + + // List the elements that you might actually get + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string elemname = (*f_iter)->get_name() + "_response"; + t_type* returntype = (*f_iter)->get_returntype(); + generate_element(f_xsd_, elemname, returntype); + f_xsd_ << endl; + + t_struct* xs = (*f_iter)->get_xceptions(); + const std::vector& xceptions = xs->get_members(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + all_xceptions[(*x_iter)->get_name()] = (t_struct*)((*x_iter)->get_type()); + } + } + + map::iterator ax_iter; + for (ax_iter = all_xceptions.begin(); ax_iter != all_xceptions.end(); ++ax_iter) { + generate_element(f_xsd_, ax_iter->first, ax_iter->second); + } + + // Close the XSD document + f_xsd_ << endl << "" << endl; + f_xsd_.close(); +} + +string t_xsd_generator::type_name(t_type* ttype) { + if (ttype->is_typedef()) { + return ttype->get_name(); + } + + if (ttype->is_base_type()) { + return xsd(base_type_name(((t_base_type*)ttype)->get_base())); + } + + if (ttype->is_enum()) { + return xsd("int"); + } + + if (ttype->is_struct() || ttype->is_xception()) { + return ttype->get_name(); + } + + return "container"; +} + +/** + * Returns the XSD type that corresponds to the thrift type. + * + * @param tbase The base type + * @return Explicit XSD type, i.e. xsd:string + */ +string t_xsd_generator::base_type_name(t_base_type::t_base tbase) { + switch (tbase) { + case t_base_type::TYPE_VOID: + return "void"; + case t_base_type::TYPE_STRING: + return "string"; + case t_base_type::TYPE_BOOL: + return "boolean"; + case t_base_type::TYPE_BYTE: + return "byte"; + case t_base_type::TYPE_I16: + return "short"; + case t_base_type::TYPE_I32: + return "int"; + case t_base_type::TYPE_I64: + return "long"; + case t_base_type::TYPE_DOUBLE: + return "decimal"; + default: + throw "compiler error: no C++ base type name for base type " + t_base_type::t_base_name(tbase); + } +} + +THRIFT_REGISTER_GENERATOR(xsd, "XSD", ""); diff --git a/compiler/cpp/src/globals.h b/compiler/cpp/src/globals.h new file mode 100644 index 00000000..b2041436 --- /dev/null +++ b/compiler/cpp/src/globals.h @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_GLOBALS_H +#define T_GLOBALS_H + +#include +#include +#include +#include +#include + +/** + * This module contains all the global variables (slap on the wrist) that are + * shared throughout the program. The reason for this is to facilitate simple + * interaction between the parser and the rest of the program. Before calling + * yyparse(), the main.cc program will make necessary adjustments to these + * global variables such that the parser does the right thing and puts entries + * into the right containers, etc. + * + */ + +/** + * Hooray for forward declaration of types! + */ + +class t_program; +class t_scope; +class t_type; + +/** + * Parsing mode, two passes up in this gin rummy! + */ + +enum PARSE_MODE { + INCLUDES = 1, + PROGRAM = 2 +}; + +/** + * Strictness level + */ +extern int g_strict; + +/** + * The master program parse tree. This is accessed from within the parser code + * to build up the program elements. + */ +extern t_program* g_program; + +/** + * Global types for the parser to be able to reference + */ + +extern t_type* g_type_void; +extern t_type* g_type_string; +extern t_type* g_type_binary; +extern t_type* g_type_slist; +extern t_type* g_type_bool; +extern t_type* g_type_byte; +extern t_type* g_type_i16; +extern t_type* g_type_i32; +extern t_type* g_type_i64; +extern t_type* g_type_double; + +/** + * The scope that we are currently parsing into + */ +extern t_scope* g_scope; + +/** + * The parent scope to also load symbols into + */ +extern t_scope* g_parent_scope; + +/** + * The prefix for the parent scope entries + */ +extern std::string g_parent_prefix; + +/** + * The parsing pass that we are on. We do different things on each pass. + */ +extern PARSE_MODE g_parse_mode; + +/** + * Global time string, used in formatting error messages etc. + */ +extern char* g_time_str; + +/** + * The last parsed doctext comment. + */ +extern char* g_doctext; + +/** + * The location of the last parsed doctext comment. + */ +extern int g_doctext_lineno; + +#endif diff --git a/compiler/cpp/src/main.cc b/compiler/cpp/src/main.cc new file mode 100644 index 00000000..7a5d2d49 --- /dev/null +++ b/compiler/cpp/src/main.cc @@ -0,0 +1,1207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * thrift - a lightweight cross-language rpc/serialization tool + * + * This file contains the main compiler engine for Thrift, which invokes the + * scanner/parser to build the thrift object tree. The interface generation + * code for each language lives in a file by the language name under the + * generate/ folder, and all parse structures live in parse/ + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef MINGW +# include /* for GetFullPathName */ +#endif + +// Careful: must include globals first for extern definitions +#include "globals.h" + +#include "main.h" +#include "parse/t_program.h" +#include "parse/t_scope.h" +#include "generate/t_generator.h" + +#include "version.h" + +using namespace std; + +/** + * Global program tree + */ +t_program* g_program; + +/** + * Global types + */ + +t_type* g_type_void; +t_type* g_type_string; +t_type* g_type_binary; +t_type* g_type_slist; +t_type* g_type_bool; +t_type* g_type_byte; +t_type* g_type_i16; +t_type* g_type_i32; +t_type* g_type_i64; +t_type* g_type_double; + +/** + * Global scope + */ +t_scope* g_scope; + +/** + * Parent scope to also parse types + */ +t_scope* g_parent_scope; + +/** + * Prefix for putting types in parent scope + */ +string g_parent_prefix; + +/** + * Parsing pass + */ +PARSE_MODE g_parse_mode; + +/** + * Current directory of file being parsed + */ +string g_curdir; + +/** + * Current file being parsed + */ +string g_curpath; + +/** + * Search path for inclusions + */ +vector g_incl_searchpath; + +/** + * Should C++ include statements use path prefixes for other thrift-generated + * header files + */ +bool g_cpp_use_include_prefix = false; + +/** + * Global debug state + */ +int g_debug = 0; + +/** + * Strictness level + */ +int g_strict = 127; + +/** + * Warning level + */ +int g_warn = 1; + +/** + * Verbose output + */ +int g_verbose = 0; + +/** + * Global time string + */ +char* g_time_str; + +/** + * The last parsed doctext comment. + */ +char* g_doctext; + +/** + * The location of the last parsed doctext comment. + */ +int g_doctext_lineno; + +/** + * Flags to control code generation + */ +bool gen_cpp = false; +bool gen_dense = false; +bool gen_java = false; +bool gen_javabean = false; +bool gen_rb = false; +bool gen_py = false; +bool gen_py_newstyle = false; +bool gen_xsd = false; +bool gen_php = false; +bool gen_phpi = false; +bool gen_phps = true; +bool gen_phpa = false; +bool gen_phpo = false; +bool gen_rest = false; +bool gen_perl = false; +bool gen_erl = false; +bool gen_ocaml = false; +bool gen_hs = false; +bool gen_cocoa = false; +bool gen_csharp = false; +bool gen_st = false; +bool gen_recurse = false; + +/** + * MinGW doesn't have realpath, so use fallback implementation in that case, + * otherwise this just calls through to realpath + */ +char *saferealpath(const char *path, char *resolved_path) { +#ifdef MINGW + char buf[MAX_PATH]; + char* basename; + DWORD len = GetFullPathName(path, MAX_PATH, buf, &basename); + if (len == 0 || len > MAX_PATH - 1){ + strcpy(resolved_path, path); + } else { + CharLowerBuff(buf, len); + strcpy(resolved_path, buf); + } + return resolved_path; +#else + return realpath(path, resolved_path); +#endif +} + + +/** + * Report an error to the user. This is called yyerror for historical + * reasons (lex and yacc expect the error reporting routine to be called + * this). Call this function to report any errors to the user. + * yyerror takes printf style arguments. + * + * @param fmt C format string followed by additional arguments + */ +void yyerror(const char* fmt, ...) { + va_list args; + fprintf(stderr, + "[ERROR:%s:%d] (last token was '%s')\n", + g_curpath.c_str(), + yylineno, + yytext); + + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + + fprintf(stderr, "\n"); +} + +/** + * Prints a debug message from the parser. + * + * @param fmt C format string followed by additional arguments + */ +void pdebug(const char* fmt, ...) { + if (g_debug == 0) { + return; + } + va_list args; + printf("[PARSE:%d] ", yylineno); + va_start(args, fmt); + vprintf(fmt, args); + va_end(args); + printf("\n"); +} + +/** + * Prints a verbose output mode message + * + * @param fmt C format string followed by additional arguments + */ +void pverbose(const char* fmt, ...) { + if (g_verbose == 0) { + return; + } + va_list args; + va_start(args, fmt); + vprintf(fmt, args); + va_end(args); +} + +/** + * Prints a warning message + * + * @param fmt C format string followed by additional arguments + */ +void pwarning(int level, const char* fmt, ...) { + if (g_warn < level) { + return; + } + va_list args; + printf("[WARNING:%s:%d] ", g_curpath.c_str(), yylineno); + va_start(args, fmt); + vprintf(fmt, args); + va_end(args); + printf("\n"); +} + +/** + * Prints a failure message and exits + * + * @param fmt C format string followed by additional arguments + */ +void failure(const char* fmt, ...) { + va_list args; + fprintf(stderr, "[FAILURE:%s:%d] ", g_curpath.c_str(), yylineno); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + printf("\n"); + exit(1); +} + +/** + * Converts a string filename into a thrift program name + */ +string program_name(string filename) { + string::size_type slash = filename.rfind("/"); + if (slash != string::npos) { + filename = filename.substr(slash+1); + } + string::size_type dot = filename.rfind("."); + if (dot != string::npos) { + filename = filename.substr(0, dot); + } + return filename; +} + +/** + * Gets the directory path of a filename + */ +string directory_name(string filename) { + string::size_type slash = filename.rfind("/"); + // No slash, just use the current directory + if (slash == string::npos) { + return "."; + } + return filename.substr(0, slash); +} + +/** + * Finds the appropriate file path for the given filename + */ +string include_file(string filename) { + // Absolute path? Just try that + if (filename[0] == '/') { + // Realpath! + char rp[PATH_MAX]; + if (saferealpath(filename.c_str(), rp) == NULL) { + pwarning(0, "Cannot open include file %s\n", filename.c_str()); + return std::string(); + } + + // Stat this file + struct stat finfo; + if (stat(rp, &finfo) == 0) { + return rp; + } + } else { // relative path, start searching + // new search path with current dir global + vector sp = g_incl_searchpath; + sp.insert(sp.begin(), g_curdir); + + // iterate through paths + vector::iterator it; + for (it = sp.begin(); it != sp.end(); it++) { + string sfilename = *(it) + "/" + filename; + + // Realpath! + char rp[PATH_MAX]; + if (saferealpath(sfilename.c_str(), rp) == NULL) { + continue; + } + + // Stat this files + struct stat finfo; + if (stat(rp, &finfo) == 0) { + return rp; + } + } + } + + // Uh oh + pwarning(0, "Could not find include file %s\n", filename.c_str()); + return std::string(); +} + +/** + * Clears any previously stored doctext string. + * Also prints a warning if we are discarding information. + */ +void clear_doctext() { + if (g_doctext != NULL) { + pwarning(2, "Uncaptured doctext at on line %d.", g_doctext_lineno); + } + free(g_doctext); + g_doctext = NULL; +} + +/** + * Cleans up text commonly found in doxygen-like comments + * + * Warning: if you mix tabs and spaces in a non-uniform way, + * you will get what you deserve. + */ +char* clean_up_doctext(char* doctext) { + // Convert to C++ string, and remove Windows's carriage returns. + string docstring = doctext; + docstring.erase( + remove(docstring.begin(), docstring.end(), '\r'), + docstring.end()); + + // Separate into lines. + vector lines; + string::size_type pos = string::npos; + string::size_type last; + while (true) { + last = (pos == string::npos) ? 0 : pos+1; + pos = docstring.find('\n', last); + if (pos == string::npos) { + // First bit of cleaning. If the last line is only whitespace, drop it. + string::size_type nonwhite = docstring.find_first_not_of(" \t", last); + if (nonwhite != string::npos) { + lines.push_back(docstring.substr(last)); + } + break; + } + lines.push_back(docstring.substr(last, pos-last)); + } + + // A very profound docstring. + if (lines.empty()) { + return NULL; + } + + // Clear leading whitespace from the first line. + pos = lines.front().find_first_not_of(" \t"); + lines.front().erase(0, pos); + + // If every nonblank line after the first has the same number of spaces/tabs, + // then a star, remove them. + bool have_prefix = true; + bool found_prefix = false; + string::size_type prefix_len = 0; + vector::iterator l_iter; + for (l_iter = lines.begin()+1; l_iter != lines.end(); ++l_iter) { + if (l_iter->empty()) { + continue; + } + + pos = l_iter->find_first_not_of(" \t"); + if (!found_prefix) { + if (pos != string::npos) { + if (l_iter->at(pos) == '*') { + found_prefix = true; + prefix_len = pos; + } else { + have_prefix = false; + break; + } + } else { + // Whitespace-only line. Truncate it. + l_iter->clear(); + } + } else if (l_iter->size() > pos + && l_iter->at(pos) == '*' + && pos == prefix_len) { + // Business as usual. + } else if (pos == string::npos) { + // Whitespace-only line. Let's truncate it for them. + l_iter->clear(); + } else { + // The pattern has been broken. + have_prefix = false; + break; + } + } + + // If our prefix survived, delete it from every line. + if (have_prefix) { + // Get the star too. + prefix_len++; + for (l_iter = lines.begin()+1; l_iter != lines.end(); ++l_iter) { + l_iter->erase(0, prefix_len); + } + } + + // Now delete the minimum amount of leading whitespace from each line. + prefix_len = string::npos; + for (l_iter = lines.begin()+1; l_iter != lines.end(); ++l_iter) { + if (l_iter->empty()) { + continue; + } + pos = l_iter->find_first_not_of(" \t"); + if (pos != string::npos + && (prefix_len == string::npos || pos < prefix_len)) { + prefix_len = pos; + } + } + + // If our prefix survived, delete it from every line. + if (prefix_len != string::npos) { + for (l_iter = lines.begin()+1; l_iter != lines.end(); ++l_iter) { + l_iter->erase(0, prefix_len); + } + } + + // Remove trailing whitespace from every line. + for (l_iter = lines.begin(); l_iter != lines.end(); ++l_iter) { + pos = l_iter->find_last_not_of(" \t"); + if (pos != string::npos && pos != l_iter->length()-1) { + l_iter->erase(pos+1); + } + } + + // If the first line is empty, remove it. + // Don't do this earlier because a lot of steps skip the first line. + if (lines.front().empty()) { + lines.erase(lines.begin()); + } + + // Now rejoin the lines and copy them back into doctext. + docstring.clear(); + for (l_iter = lines.begin(); l_iter != lines.end(); ++l_iter) { + docstring += *l_iter; + docstring += '\n'; + } + + assert(docstring.length() <= strlen(doctext)); + strcpy(doctext, docstring.c_str()); + return doctext; +} + +/** Set to true to debug docstring parsing */ +static bool dump_docs = false; + +/** + * Dumps docstrings to stdout + * Only works for top-level definitions and the whole program doc + * (i.e., not enum constants, struct fields, or functions. + */ +void dump_docstrings(t_program* program) { + string progdoc = program->get_doc(); + if (!progdoc.empty()) { + printf("Whole program doc:\n%s\n", progdoc.c_str()); + } + const vector& typedefs = program->get_typedefs(); + vector::const_iterator t_iter; + for (t_iter = typedefs.begin(); t_iter != typedefs.end(); ++t_iter) { + t_typedef* td = *t_iter; + if (td->has_doc()) { + printf("typedef %s:\n%s\n", td->get_name().c_str(), td->get_doc().c_str()); + } + } + const vector& enums = program->get_enums(); + vector::const_iterator e_iter; + for (e_iter = enums.begin(); e_iter != enums.end(); ++e_iter) { + t_enum* en = *e_iter; + if (en->has_doc()) { + printf("enum %s:\n%s\n", en->get_name().c_str(), en->get_doc().c_str()); + } + } + const vector& consts = program->get_consts(); + vector::const_iterator c_iter; + for (c_iter = consts.begin(); c_iter != consts.end(); ++c_iter) { + t_const* co = *c_iter; + if (co->has_doc()) { + printf("const %s:\n%s\n", co->get_name().c_str(), co->get_doc().c_str()); + } + } + const vector& structs = program->get_structs(); + vector::const_iterator s_iter; + for (s_iter = structs.begin(); s_iter != structs.end(); ++s_iter) { + t_struct* st = *s_iter; + if (st->has_doc()) { + printf("struct %s:\n%s\n", st->get_name().c_str(), st->get_doc().c_str()); + } + } + const vector& xceptions = program->get_xceptions(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + t_struct* xn = *x_iter; + if (xn->has_doc()) { + printf("xception %s:\n%s\n", xn->get_name().c_str(), xn->get_doc().c_str()); + } + } + const vector& services = program->get_services(); + vector::const_iterator v_iter; + for (v_iter = services.begin(); v_iter != services.end(); ++v_iter) { + t_service* sv = *v_iter; + if (sv->has_doc()) { + printf("service %s:\n%s\n", sv->get_name().c_str(), sv->get_doc().c_str()); + } + } +} + +/** + * Call generate_fingerprint for every structure and enum. + */ +void generate_all_fingerprints(t_program* program) { + const vector& structs = program->get_structs(); + vector::const_iterator s_iter; + for (s_iter = structs.begin(); s_iter != structs.end(); ++s_iter) { + t_struct* st = *s_iter; + st->generate_fingerprint(); + } + + const vector& xceptions = program->get_xceptions(); + vector::const_iterator x_iter; + for (x_iter = xceptions.begin(); x_iter != xceptions.end(); ++x_iter) { + t_struct* st = *x_iter; + st->generate_fingerprint(); + } + + const vector& enums = program->get_enums(); + vector::const_iterator e_iter; + for (e_iter = enums.begin(); e_iter != enums.end(); ++e_iter) { + t_enum* e = *e_iter; + e->generate_fingerprint(); + } + + g_type_void->generate_fingerprint(); + + // If you want to generate fingerprints for implicit structures, start here. + /* + const vector& services = program->get_services(); + vector::const_iterator v_iter; + for (v_iter = services.begin(); v_iter != services.end(); ++v_iter) { + t_service* sv = *v_iter; + } + */ +} + +/** + * Prints the version number + */ +void version() { + printf("Thrift version %s-%s\n", THRIFT_VERSION, THRIFT_REVISION); +} + +/** + * Diplays the usage message and then exits with an error code. + */ +void usage() { + fprintf(stderr, "Usage: thrift [options] file\n"); + fprintf(stderr, "Options:\n"); + fprintf(stderr, " -version Print the compiler version\n"); + fprintf(stderr, " -o dir Set the output directory for gen-* packages\n"); + fprintf(stderr, " (default: current directory)\n"); + fprintf(stderr, " -I dir Add a directory to the list of directories\n"); + fprintf(stderr, " searched for include directives\n"); + fprintf(stderr, " -nowarn Suppress all compiler warnings (BAD!)\n"); + fprintf(stderr, " -strict Strict compiler warnings on\n"); + fprintf(stderr, " -v[erbose] Verbose mode\n"); + fprintf(stderr, " -r[ecurse] Also generate included files\n"); + fprintf(stderr, " -debug Parse debug trace to stdout\n"); + fprintf(stderr, " --gen STR Generate code with a dynamically-registered generator.\n"); + fprintf(stderr, " STR has the form language[:key1=val1[,key2,[key3=val3]]].\n"); + fprintf(stderr, " Keys and values are options passed to the generator.\n"); + fprintf(stderr, " Many options will not require values.\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "Available generators (and options):\n"); + + t_generator_registry::gen_map_t gen_map = t_generator_registry::get_generator_map(); + t_generator_registry::gen_map_t::iterator iter; + for (iter = gen_map.begin(); iter != gen_map.end(); ++iter) { + fprintf(stderr, " %s (%s):\n", + iter->second->get_short_name().c_str(), + iter->second->get_long_name().c_str()); + fprintf(stderr, "%s", iter->second->get_documentation().c_str()); + } + exit(1); +} + +/** + * You know, when I started working on Thrift I really thought it wasn't going + * to become a programming language because it was just a generator and it + * wouldn't need runtime type information and all that jazz. But then we + * decided to add constants, and all of a sudden that means runtime type + * validation and inference, except the "runtime" is the code generator + * runtime. Shit. I've been had. + */ +void validate_const_rec(std::string name, t_type* type, t_const_value* value) { + if (type->is_void()) { + throw "type error: cannot declare a void const: " + name; + } + + if (type->is_base_type()) { + t_base_type::t_base tbase = ((t_base_type*)type)->get_base(); + switch (tbase) { + case t_base_type::TYPE_STRING: + if (value->get_type() != t_const_value::CV_STRING) { + throw "type error: const \"" + name + "\" was declared as string"; + } + break; + case t_base_type::TYPE_BOOL: + if (value->get_type() != t_const_value::CV_INTEGER) { + throw "type error: const \"" + name + "\" was declared as bool"; + } + break; + case t_base_type::TYPE_BYTE: + if (value->get_type() != t_const_value::CV_INTEGER) { + throw "type error: const \"" + name + "\" was declared as byte"; + } + break; + case t_base_type::TYPE_I16: + if (value->get_type() != t_const_value::CV_INTEGER) { + throw "type error: const \"" + name + "\" was declared as i16"; + } + break; + case t_base_type::TYPE_I32: + if (value->get_type() != t_const_value::CV_INTEGER) { + throw "type error: const \"" + name + "\" was declared as i32"; + } + break; + case t_base_type::TYPE_I64: + if (value->get_type() != t_const_value::CV_INTEGER) { + throw "type error: const \"" + name + "\" was declared as i64"; + } + break; + case t_base_type::TYPE_DOUBLE: + if (value->get_type() != t_const_value::CV_INTEGER && + value->get_type() != t_const_value::CV_DOUBLE) { + throw "type error: const \"" + name + "\" was declared as double"; + } + break; + default: + throw "compiler error: no const of base type " + t_base_type::t_base_name(tbase) + name; + } + } else if (type->is_enum()) { + if (value->get_type() != t_const_value::CV_INTEGER) { + throw "type error: const \"" + name + "\" was declared as enum"; + } + } else if (type->is_struct() || type->is_xception()) { + if (value->get_type() != t_const_value::CV_MAP) { + throw "type error: const \"" + name + "\" was declared as struct/xception"; + } + const vector& fields = ((t_struct*)type)->get_members(); + vector::const_iterator f_iter; + + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + if (v_iter->first->get_type() != t_const_value::CV_STRING) { + throw "type error: " + name + " struct key must be string"; + } + t_type* field_type = NULL; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { + if ((*f_iter)->get_name() == v_iter->first->get_string()) { + field_type = (*f_iter)->get_type(); + } + } + if (field_type == NULL) { + throw "type error: " + type->get_name() + " has no field " + v_iter->first->get_string(); + } + + validate_const_rec(name + "." + v_iter->first->get_string(), field_type, v_iter->second); + } + } else if (type->is_map()) { + t_type* k_type = ((t_map*)type)->get_key_type(); + t_type* v_type = ((t_map*)type)->get_val_type(); + const map& val = value->get_map(); + map::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + validate_const_rec(name + "", k_type, v_iter->first); + validate_const_rec(name + "", v_type, v_iter->second); + } + } else if (type->is_list() || type->is_set()) { + t_type* e_type; + if (type->is_list()) { + e_type = ((t_list*)type)->get_elem_type(); + } else { + e_type = ((t_set*)type)->get_elem_type(); + } + const vector& val = value->get_list(); + vector::const_iterator v_iter; + for (v_iter = val.begin(); v_iter != val.end(); ++v_iter) { + validate_const_rec(name + "", e_type, *v_iter); + } + } +} + +/** + * Check the type of the parsed const information against its declared type + */ +void validate_const_type(t_const* c) { + validate_const_rec(c->get_name(), c->get_type(), c->get_value()); +} + +/** + * Check the type of a default value assigned to a field. + */ +void validate_field_value(t_field* field, t_const_value* cv) { + validate_const_rec(field->get_name(), field->get_type(), cv); +} + +/** + * Check that all the elements of a throws block are actually exceptions. + */ +bool validate_throws(t_struct* throws) { + const vector& members = throws->get_members(); + vector::const_iterator m_iter; + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + if (!(*m_iter)->get_type()->is_xception()) { + return false; + } + } + return true; +} + +/** + * Parses a program + */ +void parse(t_program* program, t_program* parent_program) { + // Get scope file path + string path = program->get_path(); + + // Set current dir global, which is used in the include_file function + g_curdir = directory_name(path); + g_curpath = path; + + // Open the file + yyin = fopen(path.c_str(), "r"); + if (yyin == 0) { + failure("Could not open input file: \"%s\"", path.c_str()); + } + + // Create new scope and scan for includes + pverbose("Scanning %s for includes\n", path.c_str()); + g_parse_mode = INCLUDES; + g_program = program; + g_scope = program->scope(); + try { + yylineno = 1; + if (yyparse() != 0) { + failure("Parser error during include pass."); + } + } catch (string x) { + failure(x.c_str()); + } + fclose(yyin); + + // Recursively parse all the include programs + vector& includes = program->get_includes(); + vector::iterator iter; + for (iter = includes.begin(); iter != includes.end(); ++iter) { + parse(*iter, program); + } + + // Parse the program file + g_parse_mode = PROGRAM; + g_program = program; + g_scope = program->scope(); + g_parent_scope = (parent_program != NULL) ? parent_program->scope() : NULL; + g_parent_prefix = program->get_name() + "."; + g_curpath = path; + yyin = fopen(path.c_str(), "r"); + if (yyin == 0) { + failure("Could not open input file: \"%s\"", path.c_str()); + } + pverbose("Parsing %s for types\n", path.c_str()); + yylineno = 1; + try { + if (yyparse() != 0) { + failure("Parser error during types pass."); + } + } catch (string x) { + failure(x.c_str()); + } + fclose(yyin); +} + +/** + * Generate code + */ +void generate(t_program* program, const vector& generator_strings) { + // Oooohh, recursive code generation, hot!! + if (gen_recurse) { + const vector& includes = program->get_includes(); + for (size_t i = 0; i < includes.size(); ++i) { + // Propogate output path from parent to child programs + includes[i]->set_out_path(program->get_out_path()); + + generate(includes[i], generator_strings); + } + } + + // Generate code! + try { + pverbose("Program: %s\n", program->get_path().c_str()); + + // Compute fingerprints. + generate_all_fingerprints(program); + + if (dump_docs) { + dump_docstrings(program); + } + + vector::const_iterator iter; + for (iter = generator_strings.begin(); iter != generator_strings.end(); ++iter) { + t_generator* generator = t_generator_registry::get_generator(program, *iter); + + if (generator == NULL) { + pwarning(1, "Unable to get a generator for \"%s\".\n", iter->c_str()); + } else { + pverbose("Generating \"%s\"\n", iter->c_str()); + generator->generate_program(); + delete generator; + } + } + + } catch (string s) { + printf("Error: %s\n", s.c_str()); + } catch (const char* exc) { + printf("Error: %s\n", exc); + } + +} + +/** + * Parse it up.. then spit it back out, in pretty much every language. Alright + * not that many languages, but the cool ones that we care about. + */ +int main(int argc, char** argv) { + int i; + std::string out_path; + + // Setup time string + time_t now = time(NULL); + g_time_str = ctime(&now); + + // Check for necessary arguments, you gotta have at least a filename and + // an output language flag + if (argc < 2) { + usage(); + } + + vector generator_strings; + + // Set the current path to a dummy value to make warning messages clearer. + g_curpath = "arguments"; + + // Hacky parameter handling... I didn't feel like using a library sorry! + for (i = 1; i < argc-1; i++) { + char* arg; + + arg = strtok(argv[i], " "); + while (arg != NULL) { + // Treat double dashes as single dashes + if (arg[0] == '-' && arg[1] == '-') { + ++arg; + } + + if (strcmp(arg, "-version") == 0) { + version(); + exit(1); + } else if (strcmp(arg, "-debug") == 0) { + g_debug = 1; + } else if (strcmp(arg, "-nowarn") == 0) { + g_warn = 0; + } else if (strcmp(arg, "-strict") == 0) { + g_strict = 255; + g_warn = 2; + } else if (strcmp(arg, "-v") == 0 || strcmp(arg, "-verbose") == 0 ) { + g_verbose = 1; + } else if (strcmp(arg, "-r") == 0 || strcmp(arg, "-recurse") == 0 ) { + gen_recurse = true; + } else if (strcmp(arg, "-gen") == 0) { + arg = argv[++i]; + if (arg == NULL) { + fprintf(stderr, "!!! Missing generator specification\n"); + usage(); + } + generator_strings.push_back(arg); + } else if (strcmp(arg, "-dense") == 0) { + gen_dense = true; + } else if (strcmp(arg, "-cpp") == 0) { + gen_cpp = true; + } else if (strcmp(arg, "-javabean") == 0) { + gen_javabean = true; + } else if (strcmp(arg, "-java") == 0) { + gen_java = true; + } else if (strcmp(arg, "-php") == 0) { + gen_php = true; + } else if (strcmp(arg, "-phpi") == 0) { + gen_phpi = true; + } else if (strcmp(arg, "-phps") == 0) { + gen_php = true; + gen_phps = true; + } else if (strcmp(arg, "-phpl") == 0) { + gen_php = true; + gen_phps = false; + } else if (strcmp(arg, "-phpa") == 0) { + gen_php = true; + gen_phps = false; + gen_phpa = true; + } else if (strcmp(arg, "-phpo") == 0) { + gen_php = true; + gen_phpo = true; + } else if (strcmp(arg, "-rest") == 0) { + gen_rest = true; + } else if (strcmp(arg, "-py") == 0) { + gen_py = true; + } else if (strcmp(arg, "-pyns") == 0) { + gen_py = true; + gen_py_newstyle = true; + } else if (strcmp(arg, "-rb") == 0) { + gen_rb = true; + } else if (strcmp(arg, "-xsd") == 0) { + gen_xsd = true; + } else if (strcmp(arg, "-perl") == 0) { + gen_perl = true; + } else if (strcmp(arg, "-erl") == 0) { + gen_erl = true; + } else if (strcmp(arg, "-ocaml") == 0) { + gen_ocaml = true; + } else if (strcmp(arg, "-hs") == 0) { + gen_hs = true; + } else if (strcmp(arg, "-cocoa") == 0) { + gen_cocoa = true; + } else if (strcmp(arg, "-st") == 0) { + gen_st = true; + } else if (strcmp(arg, "-csharp") == 0) { + gen_csharp = true; + } else if (strcmp(arg, "-cpp_use_include_prefix") == 0) { + g_cpp_use_include_prefix = true; + } else if (strcmp(arg, "-I") == 0) { + // An argument of "-I\ asdf" is invalid and has unknown results + arg = argv[++i]; + + if (arg == NULL) { + fprintf(stderr, "!!! Missing Include directory\n"); + usage(); + } + g_incl_searchpath.push_back(arg); + } else if (strcmp(arg, "-o") == 0) { + arg = argv[++i]; + if (arg == NULL) { + fprintf(stderr, "-o: missing output directory\n"); + usage(); + } + out_path = arg; + +#ifdef MINGW + //strip out trailing \ on Windows + int last = out_path.length()-1; + if (out_path[last] == '\\') + { + out_path.erase(last); + } +#endif + + struct stat sb; + if (stat(out_path.c_str(), &sb) < 0) { + fprintf(stderr, "Output directory %s is unusable: %s\n", out_path.c_str(), strerror(errno)); + return -1; + } + if (! S_ISDIR(sb.st_mode)) { + fprintf(stderr, "Output directory %s exists but is not a directory\n", out_path.c_str()); + return -1; + } + } else { + fprintf(stderr, "!!! Unrecognized option: %s\n", arg); + usage(); + } + + // Tokenize more + arg = strtok(NULL, " "); + } + } + + // if you're asking for version, you have a right not to pass a file + if (strcmp(argv[argc-1], "-version") == 0) { + version(); + exit(1); + } + + // TODO(dreiss): Delete these when everyone is using the new hotness. + if (gen_cpp) { + pwarning(1, "-cpp is deprecated. Use --gen cpp"); + string gen_string = "cpp:"; + if (gen_dense) { + gen_string.append("dense,"); + } + if (g_cpp_use_include_prefix) { + gen_string.append("include_prefix,"); + } + generator_strings.push_back(gen_string); + } + if (gen_java) { + pwarning(1, "-java is deprecated. Use --gen java"); + generator_strings.push_back("java"); + } + if (gen_javabean) { + pwarning(1, "-javabean is deprecated. Use --gen java:beans"); + generator_strings.push_back("java:beans"); + } + if (gen_csharp) { + pwarning(1, "-csharp is deprecated. Use --gen csharp"); + generator_strings.push_back("csharp"); + } + if (gen_py) { + pwarning(1, "-py is deprecated. Use --gen py"); + generator_strings.push_back("py"); + } + if (gen_rb) { + pwarning(1, "-rb is deprecated. Use --gen rb"); + generator_strings.push_back("rb"); + } + if (gen_perl) { + pwarning(1, "-perl is deprecated. Use --gen perl"); + generator_strings.push_back("perl"); + } + if (gen_php || gen_phpi) { + pwarning(1, "-php is deprecated. Use --gen php"); + string gen_string = "php:"; + if (gen_phpi) { + gen_string.append("inlined,"); + } else if(gen_phps) { + gen_string.append("server,"); + } else if(gen_phpa) { + gen_string.append("autoload,"); + } else if(gen_phpo) { + gen_string.append("oop,"); + } else if(gen_rest) { + gen_string.append("rest,"); + } + generator_strings.push_back(gen_string); + } + if (gen_cocoa) { + pwarning(1, "-cocoa is deprecated. Use --gen cocoa"); + generator_strings.push_back("cocoa"); + } + if (gen_erl) { + pwarning(1, "-erl is deprecated. Use --gen erl"); + generator_strings.push_back("erl"); + } + if (gen_st) { + pwarning(1, "-st is deprecated. Use --gen st"); + generator_strings.push_back("st"); + } + if (gen_ocaml) { + pwarning(1, "-ocaml is deprecated. Use --gen ocaml"); + generator_strings.push_back("ocaml"); + } + if (gen_hs) { + pwarning(1, "-hs is deprecated. Use --gen hs"); + generator_strings.push_back("hs"); + } + if (gen_xsd) { + pwarning(1, "-xsd is deprecated. Use --gen xsd"); + generator_strings.push_back("xsd"); + } + + // You gotta generate something! + if (generator_strings.empty()) { + fprintf(stderr, "!!! No output language(s) specified\n\n"); + usage(); + } + + // Real-pathify it + char rp[PATH_MAX]; + if (argv[i] == NULL) { + fprintf(stderr, "!!! Missing file name\n"); + usage(); + } + if (saferealpath(argv[i], rp) == NULL) { + failure("Could not open input file with realpath: %s", argv[i]); + } + string input_file(rp); + + // Instance of the global parse tree + t_program* program = new t_program(input_file); + if (out_path.size()) { + program->set_out_path(out_path); + } + + // Compute the cpp include prefix. + // infer this from the filename passed in + string input_filename = argv[i]; + string include_prefix; + + string::size_type last_slash = string::npos; + if ((last_slash = input_filename.rfind("/")) != string::npos) { + include_prefix = input_filename.substr(0, last_slash); + } + + program->set_include_prefix(include_prefix); + + // Initialize global types + g_type_void = new t_base_type("void", t_base_type::TYPE_VOID); + g_type_string = new t_base_type("string", t_base_type::TYPE_STRING); + g_type_binary = new t_base_type("string", t_base_type::TYPE_STRING); + ((t_base_type*)g_type_binary)->set_binary(true); + g_type_slist = new t_base_type("string", t_base_type::TYPE_STRING); + ((t_base_type*)g_type_slist)->set_string_list(true); + g_type_bool = new t_base_type("bool", t_base_type::TYPE_BOOL); + g_type_byte = new t_base_type("byte", t_base_type::TYPE_BYTE); + g_type_i16 = new t_base_type("i16", t_base_type::TYPE_I16); + g_type_i32 = new t_base_type("i32", t_base_type::TYPE_I32); + g_type_i64 = new t_base_type("i64", t_base_type::TYPE_I64); + g_type_double = new t_base_type("double", t_base_type::TYPE_DOUBLE); + + // Parse it! + parse(program, NULL); + + // The current path is not really relevant when we are doing generation. + // Reset the variable to make warning messages clearer. + g_curpath = "generation"; + // Reset yylineno for the heck of it. Use 1 instead of 0 because + // That is what shows up during argument parsing. + yylineno = 1; + + // Generate it! + generate(program, generator_strings); + + // Clean up. Who am I kidding... this program probably orphans heap memory + // all over the place, but who cares because it is about to exit and it is + // all referenced and used by this wacky parse tree up until now anyways. + + delete program; + delete g_type_void; + delete g_type_string; + delete g_type_bool; + delete g_type_byte; + delete g_type_i16; + delete g_type_i32; + delete g_type_i64; + delete g_type_double; + + // Finished + return 0; +} diff --git a/compiler/cpp/src/main.h b/compiler/cpp/src/main.h new file mode 100644 index 00000000..9b7d82d7 --- /dev/null +++ b/compiler/cpp/src/main.h @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_MAIN_H +#define T_MAIN_H + +#include +#include "parse/t_const.h" +#include "parse/t_field.h" + +/** + * Defined in the flex library + */ + +int yylex(void); + +int yyparse(void); + +/** + * Expected to be defined by Flex/Bison + */ +void yyerror(const char* fmt, ...); + +/** + * Parse debugging output, used to print helpful info + */ +void pdebug(const char* fmt, ...); + +/** + * Parser warning + */ +void pwarning(int level, const char* fmt, ...); + +/** + * Failure! + */ +void failure(const char* fmt, ...); + +/** + * Check constant types + */ +void validate_const_type(t_const* c); + +/** + * Check constant types + */ +void validate_field_value(t_field* field, t_const_value* cv); + +/** + * Check members of a throws block + */ +bool validate_throws(t_struct* throws); + +/** + * Converts a string filename into a thrift program name + */ +std::string program_name(std::string filename); + +/** + * Gets the directory path of a filename + */ +std::string directory_name(std::string filename); + +/** + * Get the absolute path for an include file + */ +std::string include_file(std::string filename); + +/** + * Clears any previously stored doctext string. + */ +void clear_doctext(); + +/** + * Cleans up text commonly found in doxygen-like comments + */ +char* clean_up_doctext(char* doctext); + +/** + * Flex utilities + */ + +extern int yylineno; +extern char yytext[]; +extern FILE* yyin; + +#endif diff --git a/compiler/cpp/src/md5.c b/compiler/cpp/src/md5.c new file mode 100644 index 00000000..c35d96c5 --- /dev/null +++ b/compiler/cpp/src/md5.c @@ -0,0 +1,381 @@ +/* + Copyright (C) 1999, 2000, 2002 Aladdin Enterprises. All rights reserved. + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + L. Peter Deutsch + ghost@aladdin.com + + */ +/* $Id: md5.c,v 1.6 2002/04/13 19:20:28 lpd Exp $ */ +/* + Independent implementation of MD5 (RFC 1321). + + This code implements the MD5 Algorithm defined in RFC 1321, whose + text is available at + http://www.ietf.org/rfc/rfc1321.txt + The code is derived from the text of the RFC, including the test suite + (section A.5) but excluding the rest of Appendix A. It does not include + any code or documentation that is identified in the RFC as being + copyrighted. + + The original and principal author of md5.c is L. Peter Deutsch + . Other authors are noted in the change history + that follows (in reverse chronological order): + + 2002-04-13 lpd Clarified derivation from RFC 1321; now handles byte order + either statically or dynamically; added missing #include + in library. + 2002-03-11 lpd Corrected argument list for main(), and added int return + type, in test program and T value program. + 2002-02-21 lpd Added missing #include in test program. + 2000-07-03 lpd Patched to eliminate warnings about "constant is + unsigned in ANSI C, signed in traditional"; made test program + self-checking. + 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. + 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5). + 1999-05-03 lpd Original version. + */ + +#include "md5.h" +#include + +#undef BYTE_ORDER /* 1 = big-endian, -1 = little-endian, 0 = unknown */ +#ifdef ARCH_IS_BIG_ENDIAN +# define BYTE_ORDER (ARCH_IS_BIG_ENDIAN ? 1 : -1) +#else +# define BYTE_ORDER 0 +#endif + +#define T_MASK ((md5_word_t)~0) +#define T1 /* 0xd76aa478 */ (T_MASK ^ 0x28955b87) +#define T2 /* 0xe8c7b756 */ (T_MASK ^ 0x173848a9) +#define T3 0x242070db +#define T4 /* 0xc1bdceee */ (T_MASK ^ 0x3e423111) +#define T5 /* 0xf57c0faf */ (T_MASK ^ 0x0a83f050) +#define T6 0x4787c62a +#define T7 /* 0xa8304613 */ (T_MASK ^ 0x57cfb9ec) +#define T8 /* 0xfd469501 */ (T_MASK ^ 0x02b96afe) +#define T9 0x698098d8 +#define T10 /* 0x8b44f7af */ (T_MASK ^ 0x74bb0850) +#define T11 /* 0xffff5bb1 */ (T_MASK ^ 0x0000a44e) +#define T12 /* 0x895cd7be */ (T_MASK ^ 0x76a32841) +#define T13 0x6b901122 +#define T14 /* 0xfd987193 */ (T_MASK ^ 0x02678e6c) +#define T15 /* 0xa679438e */ (T_MASK ^ 0x5986bc71) +#define T16 0x49b40821 +#define T17 /* 0xf61e2562 */ (T_MASK ^ 0x09e1da9d) +#define T18 /* 0xc040b340 */ (T_MASK ^ 0x3fbf4cbf) +#define T19 0x265e5a51 +#define T20 /* 0xe9b6c7aa */ (T_MASK ^ 0x16493855) +#define T21 /* 0xd62f105d */ (T_MASK ^ 0x29d0efa2) +#define T22 0x02441453 +#define T23 /* 0xd8a1e681 */ (T_MASK ^ 0x275e197e) +#define T24 /* 0xe7d3fbc8 */ (T_MASK ^ 0x182c0437) +#define T25 0x21e1cde6 +#define T26 /* 0xc33707d6 */ (T_MASK ^ 0x3cc8f829) +#define T27 /* 0xf4d50d87 */ (T_MASK ^ 0x0b2af278) +#define T28 0x455a14ed +#define T29 /* 0xa9e3e905 */ (T_MASK ^ 0x561c16fa) +#define T30 /* 0xfcefa3f8 */ (T_MASK ^ 0x03105c07) +#define T31 0x676f02d9 +#define T32 /* 0x8d2a4c8a */ (T_MASK ^ 0x72d5b375) +#define T33 /* 0xfffa3942 */ (T_MASK ^ 0x0005c6bd) +#define T34 /* 0x8771f681 */ (T_MASK ^ 0x788e097e) +#define T35 0x6d9d6122 +#define T36 /* 0xfde5380c */ (T_MASK ^ 0x021ac7f3) +#define T37 /* 0xa4beea44 */ (T_MASK ^ 0x5b4115bb) +#define T38 0x4bdecfa9 +#define T39 /* 0xf6bb4b60 */ (T_MASK ^ 0x0944b49f) +#define T40 /* 0xbebfbc70 */ (T_MASK ^ 0x4140438f) +#define T41 0x289b7ec6 +#define T42 /* 0xeaa127fa */ (T_MASK ^ 0x155ed805) +#define T43 /* 0xd4ef3085 */ (T_MASK ^ 0x2b10cf7a) +#define T44 0x04881d05 +#define T45 /* 0xd9d4d039 */ (T_MASK ^ 0x262b2fc6) +#define T46 /* 0xe6db99e5 */ (T_MASK ^ 0x1924661a) +#define T47 0x1fa27cf8 +#define T48 /* 0xc4ac5665 */ (T_MASK ^ 0x3b53a99a) +#define T49 /* 0xf4292244 */ (T_MASK ^ 0x0bd6ddbb) +#define T50 0x432aff97 +#define T51 /* 0xab9423a7 */ (T_MASK ^ 0x546bdc58) +#define T52 /* 0xfc93a039 */ (T_MASK ^ 0x036c5fc6) +#define T53 0x655b59c3 +#define T54 /* 0x8f0ccc92 */ (T_MASK ^ 0x70f3336d) +#define T55 /* 0xffeff47d */ (T_MASK ^ 0x00100b82) +#define T56 /* 0x85845dd1 */ (T_MASK ^ 0x7a7ba22e) +#define T57 0x6fa87e4f +#define T58 /* 0xfe2ce6e0 */ (T_MASK ^ 0x01d3191f) +#define T59 /* 0xa3014314 */ (T_MASK ^ 0x5cfebceb) +#define T60 0x4e0811a1 +#define T61 /* 0xf7537e82 */ (T_MASK ^ 0x08ac817d) +#define T62 /* 0xbd3af235 */ (T_MASK ^ 0x42c50dca) +#define T63 0x2ad7d2bb +#define T64 /* 0xeb86d391 */ (T_MASK ^ 0x14792c6e) + + +static void +md5_process(md5_state_t *pms, const md5_byte_t *data /*[64]*/) +{ + md5_word_t + a = pms->abcd[0], b = pms->abcd[1], + c = pms->abcd[2], d = pms->abcd[3]; + md5_word_t t; +#if BYTE_ORDER > 0 + /* Define storage only for big-endian CPUs. */ + md5_word_t X[16]; +#else + /* Define storage for little-endian or both types of CPUs. */ + md5_word_t xbuf[16]; + const md5_word_t *X; +#endif + + { +#if BYTE_ORDER == 0 + /* + * Determine dynamically whether this is a big-endian or + * little-endian machine, since we can use a more efficient + * algorithm on the latter. + */ + static const int w = 1; + + if (*((const md5_byte_t *)&w)) /* dynamic little-endian */ +#endif +#if BYTE_ORDER <= 0 /* little-endian */ + { + /* + * On little-endian machines, we can process properly aligned + * data without copying it. + */ + if (!((data - (const md5_byte_t *)0) & 3)) { + /* data are properly aligned */ + X = (const md5_word_t *)data; + } else { + /* not aligned */ + memcpy(xbuf, data, 64); + X = xbuf; + } + } +#endif +#if BYTE_ORDER == 0 + else /* dynamic big-endian */ +#endif +#if BYTE_ORDER >= 0 /* big-endian */ + { + /* + * On big-endian machines, we must arrange the bytes in the + * right order. + */ + const md5_byte_t *xp = data; + int i; + +# if BYTE_ORDER == 0 + X = xbuf; /* (dynamic only) */ +# else +# define xbuf X /* (static only) */ +# endif + for (i = 0; i < 16; ++i, xp += 4) + xbuf[i] = xp[0] + (xp[1] << 8) + (xp[2] << 16) + (xp[3] << 24); + } +#endif + } + +#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) + + /* Round 1. */ + /* Let [abcd k s i] denote the operation + a = b + ((a + F(b,c,d) + X[k] + T[i]) <<< s). */ +#define F(x, y, z) (((x) & (y)) | (~(x) & (z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + F(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 0, 7, T1); + SET(d, a, b, c, 1, 12, T2); + SET(c, d, a, b, 2, 17, T3); + SET(b, c, d, a, 3, 22, T4); + SET(a, b, c, d, 4, 7, T5); + SET(d, a, b, c, 5, 12, T6); + SET(c, d, a, b, 6, 17, T7); + SET(b, c, d, a, 7, 22, T8); + SET(a, b, c, d, 8, 7, T9); + SET(d, a, b, c, 9, 12, T10); + SET(c, d, a, b, 10, 17, T11); + SET(b, c, d, a, 11, 22, T12); + SET(a, b, c, d, 12, 7, T13); + SET(d, a, b, c, 13, 12, T14); + SET(c, d, a, b, 14, 17, T15); + SET(b, c, d, a, 15, 22, T16); +#undef SET + + /* Round 2. */ + /* Let [abcd k s i] denote the operation + a = b + ((a + G(b,c,d) + X[k] + T[i]) <<< s). */ +#define G(x, y, z) (((x) & (z)) | ((y) & ~(z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + G(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 1, 5, T17); + SET(d, a, b, c, 6, 9, T18); + SET(c, d, a, b, 11, 14, T19); + SET(b, c, d, a, 0, 20, T20); + SET(a, b, c, d, 5, 5, T21); + SET(d, a, b, c, 10, 9, T22); + SET(c, d, a, b, 15, 14, T23); + SET(b, c, d, a, 4, 20, T24); + SET(a, b, c, d, 9, 5, T25); + SET(d, a, b, c, 14, 9, T26); + SET(c, d, a, b, 3, 14, T27); + SET(b, c, d, a, 8, 20, T28); + SET(a, b, c, d, 13, 5, T29); + SET(d, a, b, c, 2, 9, T30); + SET(c, d, a, b, 7, 14, T31); + SET(b, c, d, a, 12, 20, T32); +#undef SET + + /* Round 3. */ + /* Let [abcd k s t] denote the operation + a = b + ((a + H(b,c,d) + X[k] + T[i]) <<< s). */ +#define H(x, y, z) ((x) ^ (y) ^ (z)) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + H(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 5, 4, T33); + SET(d, a, b, c, 8, 11, T34); + SET(c, d, a, b, 11, 16, T35); + SET(b, c, d, a, 14, 23, T36); + SET(a, b, c, d, 1, 4, T37); + SET(d, a, b, c, 4, 11, T38); + SET(c, d, a, b, 7, 16, T39); + SET(b, c, d, a, 10, 23, T40); + SET(a, b, c, d, 13, 4, T41); + SET(d, a, b, c, 0, 11, T42); + SET(c, d, a, b, 3, 16, T43); + SET(b, c, d, a, 6, 23, T44); + SET(a, b, c, d, 9, 4, T45); + SET(d, a, b, c, 12, 11, T46); + SET(c, d, a, b, 15, 16, T47); + SET(b, c, d, a, 2, 23, T48); +#undef SET + + /* Round 4. */ + /* Let [abcd k s t] denote the operation + a = b + ((a + I(b,c,d) + X[k] + T[i]) <<< s). */ +#define I(x, y, z) ((y) ^ ((x) | ~(z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + I(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 0, 6, T49); + SET(d, a, b, c, 7, 10, T50); + SET(c, d, a, b, 14, 15, T51); + SET(b, c, d, a, 5, 21, T52); + SET(a, b, c, d, 12, 6, T53); + SET(d, a, b, c, 3, 10, T54); + SET(c, d, a, b, 10, 15, T55); + SET(b, c, d, a, 1, 21, T56); + SET(a, b, c, d, 8, 6, T57); + SET(d, a, b, c, 15, 10, T58); + SET(c, d, a, b, 6, 15, T59); + SET(b, c, d, a, 13, 21, T60); + SET(a, b, c, d, 4, 6, T61); + SET(d, a, b, c, 11, 10, T62); + SET(c, d, a, b, 2, 15, T63); + SET(b, c, d, a, 9, 21, T64); +#undef SET + + /* Then perform the following additions. (That is increment each + of the four registers by the value it had before this block + was started.) */ + pms->abcd[0] += a; + pms->abcd[1] += b; + pms->abcd[2] += c; + pms->abcd[3] += d; +} + +void +md5_init(md5_state_t *pms) +{ + pms->count[0] = pms->count[1] = 0; + pms->abcd[0] = 0x67452301; + pms->abcd[1] = /*0xefcdab89*/ T_MASK ^ 0x10325476; + pms->abcd[2] = /*0x98badcfe*/ T_MASK ^ 0x67452301; + pms->abcd[3] = 0x10325476; +} + +void +md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes) +{ + const md5_byte_t *p = data; + int left = nbytes; + int offset = (pms->count[0] >> 3) & 63; + md5_word_t nbits = (md5_word_t)(nbytes << 3); + + if (nbytes <= 0) + return; + + /* Update the message length. */ + pms->count[1] += nbytes >> 29; + pms->count[0] += nbits; + if (pms->count[0] < nbits) + pms->count[1]++; + + /* Process an initial partial block. */ + if (offset) { + int copy = (offset + nbytes > 64 ? 64 - offset : nbytes); + + memcpy(pms->buf + offset, p, copy); + if (offset + copy < 64) + return; + p += copy; + left -= copy; + md5_process(pms, pms->buf); + } + + /* Process full blocks. */ + for (; left >= 64; p += 64, left -= 64) + md5_process(pms, p); + + /* Process a final partial block. */ + if (left) + memcpy(pms->buf, p, left); +} + +void +md5_finish(md5_state_t *pms, md5_byte_t digest[16]) +{ + static const md5_byte_t pad[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + md5_byte_t data[8]; + int i; + + /* Save the length before padding. */ + for (i = 0; i < 8; ++i) + data[i] = (md5_byte_t)(pms->count[i >> 2] >> ((i & 3) << 3)); + /* Pad to 56 bytes mod 64. */ + md5_append(pms, pad, ((55 - (pms->count[0] >> 3)) & 63) + 1); + /* Append the length. */ + md5_append(pms, data, 8); + for (i = 0; i < 16; ++i) + digest[i] = (md5_byte_t)(pms->abcd[i >> 2] >> ((i & 3) << 3)); +} diff --git a/compiler/cpp/src/md5.h b/compiler/cpp/src/md5.h new file mode 100644 index 00000000..698c995d --- /dev/null +++ b/compiler/cpp/src/md5.h @@ -0,0 +1,91 @@ +/* + Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved. + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + L. Peter Deutsch + ghost@aladdin.com + + */ +/* $Id: md5.h,v 1.4 2002/04/13 19:20:28 lpd Exp $ */ +/* + Independent implementation of MD5 (RFC 1321). + + This code implements the MD5 Algorithm defined in RFC 1321, whose + text is available at + http://www.ietf.org/rfc/rfc1321.txt + The code is derived from the text of the RFC, including the test suite + (section A.5) but excluding the rest of Appendix A. It does not include + any code or documentation that is identified in the RFC as being + copyrighted. + + The original and principal author of md5.h is L. Peter Deutsch + . Other authors are noted in the change history + that follows (in reverse chronological order): + + 2002-04-13 lpd Removed support for non-ANSI compilers; removed + references to Ghostscript; clarified derivation from RFC 1321; + now handles byte order either statically or dynamically. + 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. + 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5); + added conditionalization for C++ compilation from Martin + Purschke . + 1999-05-03 lpd Original version. + */ + +#ifndef md5_INCLUDED +# define md5_INCLUDED + +/* + * This package supports both compile-time and run-time determination of CPU + * byte order. If ARCH_IS_BIG_ENDIAN is defined as 0, the code will be + * compiled to run only on little-endian CPUs; if ARCH_IS_BIG_ENDIAN is + * defined as non-zero, the code will be compiled to run only on big-endian + * CPUs; if ARCH_IS_BIG_ENDIAN is not defined, the code will be compiled to + * run on either big- or little-endian CPUs, but will run slightly less + * efficiently on either one than if ARCH_IS_BIG_ENDIAN is defined. + */ + +typedef unsigned char md5_byte_t; /* 8-bit byte */ +typedef unsigned int md5_word_t; /* 32-bit word */ + +/* Define the state of the MD5 Algorithm. */ +typedef struct md5_state_s { + md5_word_t count[2]; /* message length in bits, lsw first */ + md5_word_t abcd[4]; /* digest buffer */ + md5_byte_t buf[64]; /* accumulate block */ +} md5_state_t; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* Initialize the algorithm. */ +void md5_init(md5_state_t *pms); + +/* Append a string to the message. */ +void md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes); + +/* Finish the message and return the digest. */ +void md5_finish(md5_state_t *pms, md5_byte_t digest[16]); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif + +#endif /* md5_INCLUDED */ diff --git a/compiler/cpp/src/parse/t_base_type.h b/compiler/cpp/src/parse/t_base_type.h new file mode 100644 index 00000000..1751df9b --- /dev/null +++ b/compiler/cpp/src/parse/t_base_type.h @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_BASE_TYPE_H +#define T_BASE_TYPE_H + +#include +#include "t_type.h" + +/** + * A thrift base type, which must be one of the defined enumerated types inside + * this definition. + * + */ +class t_base_type : public t_type { + public: + /** + * Enumeration of thrift base types + */ + enum t_base { + TYPE_VOID, + TYPE_STRING, + TYPE_BOOL, + TYPE_BYTE, + TYPE_I16, + TYPE_I32, + TYPE_I64, + TYPE_DOUBLE + }; + + t_base_type(std::string name, t_base base) : + t_type(name), + base_(base), + string_list_(false), + binary_(false), + string_enum_(false){} + + t_base get_base() const { + return base_; + } + + bool is_void() const { + return base_ == TYPE_VOID; + } + + bool is_string() const { + return base_ == TYPE_STRING; + } + + bool is_bool() const { + return base_ == TYPE_BOOL; + } + + void set_string_list(bool val) { + string_list_ = val; + } + + bool is_string_list() const { + return (base_ == TYPE_STRING) && string_list_; + } + + void set_binary(bool val) { + binary_ = val; + } + + bool is_binary() const { + return (base_ == TYPE_STRING) && binary_; + } + + void set_string_enum(bool val) { + string_enum_ = true; + } + + bool is_string_enum() const { + return base_ == TYPE_STRING && string_enum_; + } + + void add_string_enum_val(std::string val) { + string_enum_vals_.push_back(val); + } + + const std::vector& get_string_enum_vals() const { + return string_enum_vals_; + } + + bool is_base_type() const { + return true; + } + + virtual std::string get_fingerprint_material() const { + std::string rv = t_base_name(base_); + if (rv == "(unknown)") { + throw "BUG: Can't get fingerprint material for this base type."; + } + return rv; + } + + static std::string t_base_name(t_base tbase) { + switch (tbase) { + case TYPE_VOID : return "void"; break; + case TYPE_STRING : return "string"; break; + case TYPE_BOOL : return "bool"; break; + case TYPE_BYTE : return "byte"; break; + case TYPE_I16 : return "i16"; break; + case TYPE_I32 : return "i32"; break; + case TYPE_I64 : return "i64"; break; + case TYPE_DOUBLE : return "double"; break; + default : return "(unknown)"; break; + } + } + + private: + t_base base_; + + bool string_list_; + bool binary_; + bool string_enum_; + std::vector string_enum_vals_; +}; + +#endif diff --git a/compiler/cpp/src/parse/t_const.h b/compiler/cpp/src/parse/t_const.h new file mode 100644 index 00000000..7fd81bd1 --- /dev/null +++ b/compiler/cpp/src/parse/t_const.h @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_CONST_H +#define T_CONST_H + +#include "t_type.h" +#include "t_const_value.h" + +/** + * A const is a constant value defined across languages that has a type and + * a value. The trick here is that the declared type might not match the type + * of the value object, since that is not determined until after parsing the + * whole thing out. + * + */ +class t_const : public t_doc { + public: + t_const(t_type* type, std::string name, t_const_value* value) : + type_(type), + name_(name), + value_(value) {} + + t_type* get_type() const { + return type_; + } + + std::string get_name() const { + return name_; + } + + t_const_value* get_value() const { + return value_; + } + + private: + t_type* type_; + std::string name_; + t_const_value* value_; +}; + +#endif + diff --git a/compiler/cpp/src/parse/t_const_value.h b/compiler/cpp/src/parse/t_const_value.h new file mode 100644 index 00000000..a7d6e31c --- /dev/null +++ b/compiler/cpp/src/parse/t_const_value.h @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_CONST_VALUE_H +#define T_CONST_VALUE_H + +#include "t_const.h" +#include +#include +#include + +/** + * A const value is something parsed that could be a map, set, list, struct + * or whatever. + * + */ +class t_const_value { + public: + + enum t_const_value_type { + CV_INTEGER, + CV_DOUBLE, + CV_STRING, + CV_MAP, + CV_LIST + }; + + t_const_value() {} + + t_const_value(int64_t val) { + set_integer(val); + } + + t_const_value(std::string val) { + set_string(val); + } + + void set_string(std::string val) { + valType_ = CV_STRING; + stringVal_ = val; + } + + std::string get_string() const { + return stringVal_; + } + + void set_integer(int64_t val) { + valType_ = CV_INTEGER; + intVal_ = val; + } + + int64_t get_integer() const { + return intVal_; + } + + void set_double(double val) { + valType_ = CV_DOUBLE; + doubleVal_ = val; + } + + double get_double() const { + return doubleVal_; + } + + void set_map() { + valType_ = CV_MAP; + } + + void add_map(t_const_value* key, t_const_value* val) { + mapVal_[key] = val; + } + + const std::map& get_map() const { + return mapVal_; + } + + void set_list() { + valType_ = CV_LIST; + } + + void add_list(t_const_value* val) { + listVal_.push_back(val); + } + + const std::vector& get_list() const { + return listVal_; + } + + t_const_value_type get_type() const { + return valType_; + } + + private: + std::map mapVal_; + std::vector listVal_; + std::string stringVal_; + int64_t intVal_; + double doubleVal_; + + t_const_value_type valType_; + +}; + +#endif + diff --git a/compiler/cpp/src/parse/t_container.h b/compiler/cpp/src/parse/t_container.h new file mode 100644 index 00000000..6753493a --- /dev/null +++ b/compiler/cpp/src/parse/t_container.h @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_CONTAINER_H +#define T_CONTAINER_H + +#include "t_type.h" + +class t_container : public t_type { + public: + t_container() : + cpp_name_(), + has_cpp_name_(false) {} + + virtual ~t_container() {} + + void set_cpp_name(std::string cpp_name) { + cpp_name_ = cpp_name; + has_cpp_name_ = true; + } + + bool has_cpp_name() { + return has_cpp_name_; + } + + std::string get_cpp_name() { + return cpp_name_; + } + + bool is_container() const { + return true; + } + + private: + std::string cpp_name_; + bool has_cpp_name_; + +}; + +#endif diff --git a/compiler/cpp/src/parse/t_doc.h b/compiler/cpp/src/parse/t_doc.h new file mode 100644 index 00000000..e52068cb --- /dev/null +++ b/compiler/cpp/src/parse/t_doc.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_DOC_H +#define T_DOC_H + +/** + * Documentation stubs + * + */ +class t_doc { + + public: + t_doc() : has_doc_(false) {} + + void set_doc(const std::string& doc) { + doc_ = doc; + has_doc_ = true; + } + + const std::string& get_doc() const { + return doc_; + } + + bool has_doc() { + return has_doc_; + } + + private: + std::string doc_; + bool has_doc_; + +}; + +#endif diff --git a/compiler/cpp/src/parse/t_enum.h b/compiler/cpp/src/parse/t_enum.h new file mode 100644 index 00000000..740f95ca --- /dev/null +++ b/compiler/cpp/src/parse/t_enum.h @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_ENUM_H +#define T_ENUM_H + +#include "t_enum_value.h" +#include + +/** + * An enumerated type. A list of constant objects with a name for the type. + * + */ +class t_enum : public t_type { + public: + t_enum(t_program* program) : + t_type(program) {} + + void set_name(const std::string& name) { + name_ = name; + } + + void append(t_enum_value* constant) { + constants_.push_back(constant); + } + + const std::vector& get_constants() { + return constants_; + } + + bool is_enum() const { + return true; + } + + virtual std::string get_fingerprint_material() const { + return "enum"; + } + + private: + std::vector constants_; +}; + +#endif diff --git a/compiler/cpp/src/parse/t_enum_value.h b/compiler/cpp/src/parse/t_enum_value.h new file mode 100644 index 00000000..68e905bd --- /dev/null +++ b/compiler/cpp/src/parse/t_enum_value.h @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_ENUM_VALUE_H +#define T_ENUM_VALUE_H + +#include +#include "t_doc.h" + +/** + * A constant. These are used inside of enum definitions. Constants are just + * symbol identifiers that may or may not have an explicit value associated + * with them. + * + */ +class t_enum_value : public t_doc { + public: + t_enum_value(std::string name) : + name_(name), + has_value_(false), + value_(0) {} + + t_enum_value(std::string name, int value) : + name_(name), + has_value_(true), + value_(value) {} + + ~t_enum_value() {} + + const std::string& get_name() { + return name_; + } + + bool has_value() { + return has_value_; + } + + int get_value() { + return value_; + } + + private: + std::string name_; + bool has_value_; + int value_; +}; + +#endif diff --git a/compiler/cpp/src/parse/t_field.h b/compiler/cpp/src/parse/t_field.h new file mode 100644 index 00000000..67a2125c --- /dev/null +++ b/compiler/cpp/src/parse/t_field.h @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_FIELD_H +#define T_FIELD_H + +#include +#include + +#include "t_doc.h" + +// Forward declare for xsd_attrs +class t_struct; + +/** + * Class to represent a field in a thrift structure. A field has a data type, + * a symbolic name, and a numeric identifier. + * + */ +class t_field : public t_doc { + public: + t_field(t_type* type, std::string name) : + type_(type), + name_(name), + key_(0), + value_(NULL), + xsd_optional_(false), + xsd_nillable_(false), + xsd_attrs_(NULL) {} + + t_field(t_type* type, std::string name, int32_t key) : + type_(type), + name_(name), + key_(key), + req_(T_OPT_IN_REQ_OUT), + value_(NULL), + xsd_optional_(false), + xsd_nillable_(false), + xsd_attrs_(NULL) {} + + ~t_field() {} + + t_type* get_type() const { + return type_; + } + + const std::string& get_name() const { + return name_; + } + + int32_t get_key() const { + return key_; + } + + enum e_req { + T_REQUIRED, + T_OPTIONAL, + T_OPT_IN_REQ_OUT, + }; + + void set_req(e_req req) { + req_ = req; + } + + e_req get_req() const { + return req_; + } + + void set_value(t_const_value* value) { + value_ = value; + } + + t_const_value* get_value() { + return value_; + } + + void set_xsd_optional(bool xsd_optional) { + xsd_optional_ = xsd_optional; + } + + bool get_xsd_optional() const { + return xsd_optional_; + } + + void set_xsd_nillable(bool xsd_nillable) { + xsd_nillable_ = xsd_nillable; + } + + bool get_xsd_nillable() const { + return xsd_nillable_; + } + + void set_xsd_attrs(t_struct* xsd_attrs) { + xsd_attrs_ = xsd_attrs; + } + + t_struct* get_xsd_attrs() { + return xsd_attrs_; + } + + // This is not the same function as t_type::get_fingerprint_material, + // but it does the same thing. + std::string get_fingerprint_material() const { + return boost::lexical_cast(key_) + ":" + + ((req_ == T_OPTIONAL) ? "opt-" : "") + + type_->get_fingerprint_material(); + } + + /** + * Comparator to sort fields in ascending order by key. + * Make this a functor instead of a function to help GCC inline it. + * The arguments are (const) references to const pointers to const t_fields. + */ + struct key_compare { + bool operator()(t_field const * const & a, t_field const * const & b) { + return a->get_key() < b->get_key(); + } + }; + + + private: + t_type* type_; + std::string name_; + int32_t key_; + e_req req_; + t_const_value* value_; + + bool xsd_optional_; + bool xsd_nillable_; + t_struct* xsd_attrs_; + +}; + +#endif diff --git a/compiler/cpp/src/parse/t_function.h b/compiler/cpp/src/parse/t_function.h new file mode 100644 index 00000000..a72aa6c3 --- /dev/null +++ b/compiler/cpp/src/parse/t_function.h @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_FUNCTION_H +#define T_FUNCTION_H + +#include +#include "t_type.h" +#include "t_struct.h" +#include "t_doc.h" + +/** + * Representation of a function. Key parts are return type, function name, + * optional modifiers, and an argument list, which is implemented as a thrift + * struct. + * + */ +class t_function : public t_doc { + public: + t_function(t_type* returntype, + std::string name, + t_struct* arglist, + bool oneway=false) : + returntype_(returntype), + name_(name), + arglist_(arglist), + oneway_(oneway) { + xceptions_ = new t_struct(NULL); + } + + t_function(t_type* returntype, + std::string name, + t_struct* arglist, + t_struct* xceptions, + bool oneway=false) : + returntype_(returntype), + name_(name), + arglist_(arglist), + xceptions_(xceptions), + oneway_(oneway) + { + if (oneway_ && !xceptions_->get_members().empty()) { + throw std::string("Oneway methods can't throw exceptions."); + } + } + + ~t_function() {} + + t_type* get_returntype() const { + return returntype_; + } + + const std::string& get_name() const { + return name_; + } + + t_struct* get_arglist() const { + return arglist_; + } + + t_struct* get_xceptions() const { + return xceptions_; + } + + bool is_oneway() const { + return oneway_; + } + + private: + t_type* returntype_; + std::string name_; + t_struct* arglist_; + t_struct* xceptions_; + bool oneway_; +}; + +#endif diff --git a/compiler/cpp/src/parse/t_list.h b/compiler/cpp/src/parse/t_list.h new file mode 100644 index 00000000..21a9625e --- /dev/null +++ b/compiler/cpp/src/parse/t_list.h @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_LIST_H +#define T_LIST_H + +#include "t_container.h" + +/** + * A list is a lightweight container type that just wraps another data type. + * + */ +class t_list : public t_container { + public: + t_list(t_type* elem_type) : + elem_type_(elem_type) {} + + t_type* get_elem_type() const { + return elem_type_; + } + + bool is_list() const { + return true; + } + + virtual std::string get_fingerprint_material() const { + return "list<" + elem_type_->get_fingerprint_material() + ">"; + } + + virtual void generate_fingerprint() { + t_type::generate_fingerprint(); + elem_type_->generate_fingerprint(); + } + + private: + t_type* elem_type_; +}; + +#endif + diff --git a/compiler/cpp/src/parse/t_map.h b/compiler/cpp/src/parse/t_map.h new file mode 100644 index 00000000..c4e358fd --- /dev/null +++ b/compiler/cpp/src/parse/t_map.h @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_MAP_H +#define T_MAP_H + +#include "t_container.h" + +/** + * A map is a lightweight container type that just wraps another two data + * types. + * + */ +class t_map : public t_container { + public: + t_map(t_type* key_type, t_type* val_type) : + key_type_(key_type), + val_type_(val_type) {} + + t_type* get_key_type() const { + return key_type_; + } + + t_type* get_val_type() const { + return val_type_; + } + + bool is_map() const { + return true; + } + + virtual std::string get_fingerprint_material() const { + return "map<" + key_type_->get_fingerprint_material() + + "," + val_type_->get_fingerprint_material() + ">"; + } + + virtual void generate_fingerprint() { + t_type::generate_fingerprint(); + key_type_->generate_fingerprint(); + val_type_->generate_fingerprint(); + } + + private: + t_type* key_type_; + t_type* val_type_; +}; + +#endif diff --git a/compiler/cpp/src/parse/t_program.h b/compiler/cpp/src/parse/t_program.h new file mode 100644 index 00000000..4e1ab6a5 --- /dev/null +++ b/compiler/cpp/src/parse/t_program.h @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_PROGRAM_H +#define T_PROGRAM_H + +#include +#include +#include + +// For program_name() +#include "main.h" + +#include "t_doc.h" +#include "t_scope.h" +#include "t_base_type.h" +#include "t_typedef.h" +#include "t_enum.h" +#include "t_const.h" +#include "t_struct.h" +#include "t_service.h" +#include "t_list.h" +#include "t_map.h" +#include "t_set.h" +//#include "t_doc.h" + +/** + * Top level class representing an entire thrift program. A program consists + * fundamentally of the following: + * + * Typedefs + * Enumerations + * Constants + * Structs + * Exceptions + * Services + * + * The program module also contains the definitions of the base types. + * + */ +class t_program : public t_doc { + public: + t_program(std::string path, std::string name) : + path_(path), + name_(name), + out_path_("./") { + scope_ = new t_scope(); + } + + t_program(std::string path) : + path_(path), + out_path_("./") { + name_ = program_name(path); + scope_ = new t_scope(); + } + + // Path accessor + const std::string& get_path() const { return path_; } + + // Output path accessor + const std::string& get_out_path() const { return out_path_; } + + // Name accessor + const std::string& get_name() const { return name_; } + + // Namespace + const std::string& get_namespace() const { return namespace_; } + + // Include prefix accessor + const std::string& get_include_prefix() const { return include_prefix_; } + + // Accessors for program elements + const std::vector& get_typedefs() const { return typedefs_; } + const std::vector& get_enums() const { return enums_; } + const std::vector& get_consts() const { return consts_; } + const std::vector& get_structs() const { return structs_; } + const std::vector& get_xceptions() const { return xceptions_; } + const std::vector& get_objects() const { return objects_; } + const std::vector& get_services() const { return services_; } + + // Program elements + void add_typedef (t_typedef* td) { typedefs_.push_back(td); } + void add_enum (t_enum* te) { enums_.push_back(te); } + void add_const (t_const* tc) { consts_.push_back(tc); } + void add_struct (t_struct* ts) { objects_.push_back(ts); + structs_.push_back(ts); } + void add_xception (t_struct* tx) { objects_.push_back(tx); + xceptions_.push_back(tx); } + void add_service (t_service* ts) { services_.push_back(ts); } + + // Programs to include + const std::vector& get_includes() const { return includes_; } + + void set_out_path(std::string out_path) { + out_path_ = out_path; + // Ensure that it ends with a trailing '/' (or '\' for windows machines) + char c = out_path_.at(out_path_.size() - 1); + if (!(c == '/' || c == '\\')) { + out_path_.push_back('/'); + } + } + + // Scoping and namespacing + void set_namespace(std::string name) { + namespace_ = name; + } + + // Scope accessor + t_scope* scope() { + return scope_; + } + + // Includes + + void add_include(std::string path, std::string include_site) { + t_program* program = new t_program(path); + + // include prefix for this program is the site at which it was included + // (minus the filename) + std::string include_prefix; + std::string::size_type last_slash = std::string::npos; + if ((last_slash = include_site.rfind("/")) != std::string::npos) { + include_prefix = include_site.substr(0, last_slash); + } + + program->set_include_prefix(include_prefix); + includes_.push_back(program); + } + + std::vector& get_includes() { + return includes_; + } + + void set_include_prefix(std::string include_prefix) { + include_prefix_ = include_prefix; + + // this is intended to be a directory; add a trailing slash if necessary + int len = include_prefix_.size(); + if (len > 0 && include_prefix_[len - 1] != '/') { + include_prefix_ += '/'; + } + } + + // Language neutral namespace / packaging + void set_namespace(std::string language, std::string name_space) { + namespaces_[language] = name_space; + } + + std::string get_namespace(std::string language) const { + std::map::const_iterator iter = namespaces_.find(language); + if (iter == namespaces_.end()) { + return std::string(); + } + return iter->second; + } + + // Language specific namespace / packaging + + void add_cpp_include(std::string path) { + cpp_includes_.push_back(path); + } + + const std::vector& get_cpp_includes() { + return cpp_includes_; + } + + private: + + // File path + std::string path_; + + // Name + std::string name_; + + // Output directory + std::string out_path_; + + // Namespace + std::string namespace_; + + // Included programs + std::vector includes_; + + // Include prefix for this program, if any + std::string include_prefix_; + + // Identifier lookup scope + t_scope* scope_; + + // Components to generate code for + std::vector typedefs_; + std::vector enums_; + std::vector consts_; + std::vector objects_; + std::vector structs_; + std::vector xceptions_; + std::vector services_; + + // Dynamic namespaces + std::map namespaces_; + + // C++ extra includes + std::vector cpp_includes_; + +}; + +#endif diff --git a/compiler/cpp/src/parse/t_scope.h b/compiler/cpp/src/parse/t_scope.h new file mode 100644 index 00000000..122e3256 --- /dev/null +++ b/compiler/cpp/src/parse/t_scope.h @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_SCOPE_H +#define T_SCOPE_H + +#include +#include + +#include "t_type.h" +#include "t_service.h" + +/** + * This represents a variable scope used for looking up predefined types and + * services. Typically, a scope is associated with a t_program. Scopes are not + * used to determine code generation, but rather to resolve identifiers at + * parse time. + * + */ +class t_scope { + public: + t_scope() {} + + void add_type(std::string name, t_type* type) { + types_[name] = type; + } + + t_type* get_type(std::string name) { + return types_[name]; + } + + void add_service(std::string name, t_service* service) { + services_[name] = service; + } + + t_service* get_service(std::string name) { + return services_[name]; + } + + void add_constant(std::string name, t_const* constant) { + constants_[name] = constant; + } + + t_const* get_constant(std::string name) { + return constants_[name]; + } + + void print() { + std::map::iterator iter; + for (iter = types_.begin(); iter != types_.end(); ++iter) { + printf("%s => %s\n", + iter->first.c_str(), + iter->second->get_name().c_str()); + } + } + + private: + + // Map of names to types + std::map types_; + + // Map of names to constants + std::map constants_; + + // Map of names to services + std::map services_; + +}; + +#endif diff --git a/compiler/cpp/src/parse/t_service.h b/compiler/cpp/src/parse/t_service.h new file mode 100644 index 00000000..eee2dac1 --- /dev/null +++ b/compiler/cpp/src/parse/t_service.h @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_SERVICE_H +#define T_SERVICE_H + +#include "t_function.h" +#include + +class t_program; + +/** + * A service consists of a set of functions. + * + */ +class t_service : public t_type { + public: + t_service(t_program* program) : + t_type(program), + extends_(NULL) {} + + bool is_service() const { + return true; + } + + void set_extends(t_service* extends) { + extends_ = extends; + } + + void add_function(t_function* func) { + functions_.push_back(func); + } + + const std::vector& get_functions() const { + return functions_; + } + + t_service* get_extends() { + return extends_; + } + + virtual std::string get_fingerprint_material() const { + // Services should never be used in fingerprints. + throw "BUG: Can't get fingerprint material for service."; + } + + private: + std::vector functions_; + t_service* extends_; +}; + +#endif diff --git a/compiler/cpp/src/parse/t_set.h b/compiler/cpp/src/parse/t_set.h new file mode 100644 index 00000000..d1983577 --- /dev/null +++ b/compiler/cpp/src/parse/t_set.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_SET_H +#define T_SET_H + +#include "t_container.h" + +/** + * A set is a lightweight container type that just wraps another data type. + * + */ +class t_set : public t_container { + public: + t_set(t_type* elem_type) : + elem_type_(elem_type) {} + + t_type* get_elem_type() const { + return elem_type_; + } + + bool is_set() const { + return true; + } + + virtual std::string get_fingerprint_material() const { + return "set<" + elem_type_->get_fingerprint_material() + ">"; + } + + virtual void generate_fingerprint() { + t_type::generate_fingerprint(); + elem_type_->generate_fingerprint(); + } + + private: + t_type* elem_type_; +}; + +#endif diff --git a/compiler/cpp/src/parse/t_struct.h b/compiler/cpp/src/parse/t_struct.h new file mode 100644 index 00000000..7980f803 --- /dev/null +++ b/compiler/cpp/src/parse/t_struct.h @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_STRUCT_H +#define T_STRUCT_H + +#include +#include +#include +#include + +#include "t_type.h" +#include "t_field.h" + +// Forward declare that puppy +class t_program; + +/** + * A struct is a container for a set of member fields that has a name. Structs + * are also used to implement exception types. + * + */ +class t_struct : public t_type { + public: + typedef std::vector members_type; + + t_struct(t_program* program) : + t_type(program), + is_xception_(false), + xsd_all_(false) {} + + t_struct(t_program* program, const std::string& name) : + t_type(program, name), + is_xception_(false), + xsd_all_(false) {} + + void set_name(const std::string& name) { + name_ = name; + } + + void set_xception(bool is_xception) { + is_xception_ = is_xception; + } + + void set_xsd_all(bool xsd_all) { + xsd_all_ = xsd_all; + } + + bool get_xsd_all() const { + return xsd_all_; + } + + bool append(t_field* elem) { + members_.push_back(elem); + + typedef members_type::iterator iter_type; + std::pair bounds = std::equal_range( + members_in_id_order_.begin(), members_in_id_order_.end(), elem, t_field::key_compare() + ); + if (bounds.first != bounds.second) { + return false; + } + members_in_id_order_.insert(bounds.second, elem); + return true; + } + + const members_type& get_members() { + return members_; + } + + const members_type& get_sorted_members() { + return members_in_id_order_; + } + + bool is_struct() const { + return !is_xception_; + } + + bool is_xception() const { + return is_xception_; + } + + virtual std::string get_fingerprint_material() const { + std::string rv = "{"; + members_type::const_iterator m_iter; + for (m_iter = members_in_id_order_.begin(); m_iter != members_in_id_order_.end(); ++m_iter) { + rv += (*m_iter)->get_fingerprint_material(); + rv += ";"; + } + rv += "}"; + return rv; + } + + virtual void generate_fingerprint() { + t_type::generate_fingerprint(); + members_type::const_iterator m_iter; + for (m_iter = members_in_id_order_.begin(); m_iter != members_in_id_order_.end(); ++m_iter) { + (*m_iter)->get_type()->generate_fingerprint(); + } + } + + private: + + members_type members_; + members_type members_in_id_order_; + bool is_xception_; + + bool xsd_all_; +}; + +#endif diff --git a/compiler/cpp/src/parse/t_type.h b/compiler/cpp/src/parse/t_type.h new file mode 100644 index 00000000..4ce2eda1 --- /dev/null +++ b/compiler/cpp/src/parse/t_type.h @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_TYPE_H +#define T_TYPE_H + +#include +#include +#include +#include +#include "t_doc.h" + +// What's worse? This, or making a src/parse/non_inlined.cc? +#include "md5.h" + +class t_program; + +/** + * Generic representation of a thrift type. These objects are used by the + * parser module to build up a tree of object that are all explicitly typed. + * The generic t_type class exports a variety of useful methods that are + * used by the code generator to branch based upon different handling for the + * various types. + * + */ +class t_type : public t_doc { + public: + virtual ~t_type() {} + + virtual void set_name(const std::string& name) { + name_ = name; + } + + virtual const std::string& get_name() const { + return name_; + } + + virtual bool is_void() const { return false; } + virtual bool is_base_type() const { return false; } + virtual bool is_string() const { return false; } + virtual bool is_bool() const { return false; } + virtual bool is_typedef() const { return false; } + virtual bool is_enum() const { return false; } + virtual bool is_struct() const { return false; } + virtual bool is_xception() const { return false; } + virtual bool is_container() const { return false; } + virtual bool is_list() const { return false; } + virtual bool is_set() const { return false; } + virtual bool is_map() const { return false; } + virtual bool is_service() const { return false; } + + t_program* get_program() { + return program_; + } + + + // Return a string that uniquely identifies this type + // from any other thrift type in the world, as far as + // TDenseProtocol is concerned. + // We don't cache this, which is a little sloppy, + // but the compiler is so fast that it doesn't really matter. + virtual std::string get_fingerprint_material() const = 0; + + // Fingerprint should change whenever (and only when) + // the encoding via TDenseProtocol changes. + static const int fingerprint_len = 16; + + // Call this before trying get_*_fingerprint(). + virtual void generate_fingerprint() { + std::string material = get_fingerprint_material(); + md5_state_t ctx; + md5_init(&ctx); + md5_append(&ctx, (md5_byte_t*)(material.data()), (int)material.size()); + md5_finish(&ctx, (md5_byte_t*)fingerprint_); + } + + bool has_fingerprint() const { + for (int i = 0; i < fingerprint_len; i++) { + if (fingerprint_[i] != 0) { + return true; + } + } + return false; + } + + const uint8_t* get_binary_fingerprint() const { + return fingerprint_; + } + + std::string get_ascii_fingerprint() const { + std::string rv; + const uint8_t* fp = get_binary_fingerprint(); + for (int i = 0; i < fingerprint_len; i++) { + rv += byte_to_hex(fp[i]); + } + return rv; + } + + // This function will break (maybe badly) unless 0 <= num <= 16. + static char nybble_to_xdigit(int num) { + if (num < 10) { + return '0' + num; + } else { + return 'A' + num - 10; + } + } + + static std::string byte_to_hex(uint8_t byte) { + std::string rv; + rv += nybble_to_xdigit(byte >> 4); + rv += nybble_to_xdigit(byte & 0x0f); + return rv; + } + + std::map annotations_; + + protected: + t_type() : + program_(NULL) + { + memset(fingerprint_, 0, sizeof(fingerprint_)); + } + + t_type(t_program* program) : + program_(program) + { + memset(fingerprint_, 0, sizeof(fingerprint_)); + } + + t_type(t_program* program, std::string name) : + program_(program), + name_(name) + { + memset(fingerprint_, 0, sizeof(fingerprint_)); + } + + t_type(std::string name) : + program_(NULL), + name_(name) + { + memset(fingerprint_, 0, sizeof(fingerprint_)); + } + + t_program* program_; + std::string name_; + + uint8_t fingerprint_[fingerprint_len]; +}; + + +/** + * Placeholder struct for returning the key and value of an annotation + * during parsing. + */ +struct t_annotation { + std::string key; + std::string val; +}; + +#endif diff --git a/compiler/cpp/src/parse/t_typedef.h b/compiler/cpp/src/parse/t_typedef.h new file mode 100644 index 00000000..4c77d97a --- /dev/null +++ b/compiler/cpp/src/parse/t_typedef.h @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef T_TYPEDEF_H +#define T_TYPEDEF_H + +#include +#include "t_type.h" + +/** + * A typedef is a mapping from a symbolic name to another type. In dymanically + * typed languages (i.e. php/python) the code generator can actually usually + * ignore typedefs and just use the underlying type directly, though in C++ + * the symbolic naming can be quite useful for code clarity. + * + */ +class t_typedef : public t_type { + public: + t_typedef(t_program* program, t_type* type, std::string symbolic) : + t_type(program, symbolic), + type_(type), + symbolic_(symbolic) {} + + ~t_typedef() {} + + t_type* get_type() const { + return type_; + } + + const std::string& get_symbolic() const { + return symbolic_; + } + + bool is_typedef() const { + return true; + } + + virtual std::string get_fingerprint_material() const { + return type_->get_fingerprint_material(); + } + + virtual void generate_fingerprint() { + t_type::generate_fingerprint(); + if (!type_->has_fingerprint()) { + type_->generate_fingerprint(); + } + } + + private: + t_type* type_; + std::string symbolic_; +}; + +#endif diff --git a/compiler/cpp/src/platform.h b/compiler/cpp/src/platform.h new file mode 100644 index 00000000..bd97f68e --- /dev/null +++ b/compiler/cpp/src/platform.h @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * define for mkdir,since the method signature + * is different for the non-POSIX MinGW + */ + +#ifdef MINGW +#include +#else +#include +#include +#endif + +#if defined MINGW +#define MKDIR(x) mkdir(x) +#else +#define MKDIR(x) mkdir(x, S_IRWXU | S_IRWXG | S_IRWXO) +#endif diff --git a/compiler/cpp/src/thriftl.ll b/compiler/cpp/src/thriftl.ll new file mode 100644 index 00000000..2a8ab67e --- /dev/null +++ b/compiler/cpp/src/thriftl.ll @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Thrift scanner. + * + * Tokenizes a thrift definition file. + */ + +%{ + +#include +#include + +#include "main.h" +#include "globals.h" +#include "parse/t_program.h" + +/** + * Must be included AFTER parse/t_program.h, but I can't remember why anymore + * because I wrote this a while ago. + */ +#include "thrifty.h" + +void thrift_reserved_keyword(char* keyword) { + yyerror("Cannot use reserved language keyword: \"%s\"\n", keyword); + exit(1); +} + +void integer_overflow(char* text) { + yyerror("This integer is too big: \"%s\"\n", text); + exit(1); +} + +%} + +/** + * Provides the yylineno global, useful for debugging output + */ +%option lex-compat + +/** + * Helper definitions, comments, constants, and whatnot + */ + +intconstant ([+-]?[0-9]+) +hexconstant ("0x"[0-9A-Fa-f]+) +dubconstant ([+-]?[0-9]*(\.[0-9]+)?([eE][+-]?[0-9]+)?) +identifier ([a-zA-Z_][\.a-zA-Z_0-9]*) +whitespace ([ \t\r\n]*) +sillycomm ("/*""*"*"*/") +multicomm ("/*"[^*]"/"*([^*/]|[^*]"/"|"*"[^/])*"*"*"*/") +doctext ("/**"([^*/]|[^*]"/"|"*"[^/])*"*"*"*/") +comment ("//"[^\n]*) +unixcomment ("#"[^\n]*) +symbol ([:;\,\{\}\(\)\=<>\[\]]) +st_identifier ([a-zA-Z-][\.a-zA-Z_0-9-]*) +literal_begin (['\"]) + +%% + +{whitespace} { /* do nothing */ } +{sillycomm} { /* do nothing */ } +{multicomm} { /* do nothing */ } +{comment} { /* do nothing */ } +{unixcomment} { /* do nothing */ } + +{symbol} { return yytext[0]; } + +"namespace" { return tok_namespace; } +"cpp_namespace" { return tok_cpp_namespace; } +"cpp_include" { return tok_cpp_include; } +"cpp_type" { return tok_cpp_type; } +"java_package" { return tok_java_package; } +"cocoa_prefix" { return tok_cocoa_prefix; } +"csharp_namespace" { return tok_csharp_namespace; } +"php_namespace" { return tok_php_namespace; } +"py_module" { return tok_py_module; } +"perl_package" { return tok_perl_package; } +"ruby_namespace" { return tok_ruby_namespace; } +"smalltalk_category" { return tok_smalltalk_category; } +"smalltalk_prefix" { return tok_smalltalk_prefix; } +"xsd_all" { return tok_xsd_all; } +"xsd_optional" { return tok_xsd_optional; } +"xsd_nillable" { return tok_xsd_nillable; } +"xsd_namespace" { return tok_xsd_namespace; } +"xsd_attrs" { return tok_xsd_attrs; } +"include" { return tok_include; } +"void" { return tok_void; } +"bool" { return tok_bool; } +"byte" { return tok_byte; } +"i16" { return tok_i16; } +"i32" { return tok_i32; } +"i64" { return tok_i64; } +"double" { return tok_double; } +"string" { return tok_string; } +"binary" { return tok_binary; } +"slist" { return tok_slist; } +"senum" { return tok_senum; } +"map" { return tok_map; } +"list" { return tok_list; } +"set" { return tok_set; } +"oneway" { return tok_oneway; } +"typedef" { return tok_typedef; } +"struct" { return tok_struct; } +"exception" { return tok_xception; } +"extends" { return tok_extends; } +"throws" { return tok_throws; } +"service" { return tok_service; } +"enum" { return tok_enum; } +"const" { return tok_const; } +"required" { return tok_required; } +"optional" { return tok_optional; } +"async" { + pwarning(0, "\"async\" is deprecated. It is called \"oneway\" now.\n"); + return tok_oneway; +} + + +"abstract" { thrift_reserved_keyword(yytext); } +"and" { thrift_reserved_keyword(yytext); } +"args" { thrift_reserved_keyword(yytext); } +"as" { thrift_reserved_keyword(yytext); } +"assert" { thrift_reserved_keyword(yytext); } +"break" { thrift_reserved_keyword(yytext); } +"case" { thrift_reserved_keyword(yytext); } +"class" { thrift_reserved_keyword(yytext); } +"continue" { thrift_reserved_keyword(yytext); } +"declare" { thrift_reserved_keyword(yytext); } +"def" { thrift_reserved_keyword(yytext); } +"default" { thrift_reserved_keyword(yytext); } +"del" { thrift_reserved_keyword(yytext); } +"delete" { thrift_reserved_keyword(yytext); } +"do" { thrift_reserved_keyword(yytext); } +"elif" { thrift_reserved_keyword(yytext); } +"else" { thrift_reserved_keyword(yytext); } +"elseif" { thrift_reserved_keyword(yytext); } +"except" { thrift_reserved_keyword(yytext); } +"exec" { thrift_reserved_keyword(yytext); } +"false" { thrift_reserved_keyword(yytext); } +"finally" { thrift_reserved_keyword(yytext); } +"float" { thrift_reserved_keyword(yytext); } +"for" { thrift_reserved_keyword(yytext); } +"foreach" { thrift_reserved_keyword(yytext); } +"function" { thrift_reserved_keyword(yytext); } +"global" { thrift_reserved_keyword(yytext); } +"goto" { thrift_reserved_keyword(yytext); } +"if" { thrift_reserved_keyword(yytext); } +"implements" { thrift_reserved_keyword(yytext); } +"import" { thrift_reserved_keyword(yytext); } +"in" { thrift_reserved_keyword(yytext); } +"inline" { thrift_reserved_keyword(yytext); } +"instanceof" { thrift_reserved_keyword(yytext); } +"interface" { thrift_reserved_keyword(yytext); } +"is" { thrift_reserved_keyword(yytext); } +"lambda" { thrift_reserved_keyword(yytext); } +"native" { thrift_reserved_keyword(yytext); } +"new" { thrift_reserved_keyword(yytext); } +"not" { thrift_reserved_keyword(yytext); } +"or" { thrift_reserved_keyword(yytext); } +"pass" { thrift_reserved_keyword(yytext); } +"public" { thrift_reserved_keyword(yytext); } +"print" { thrift_reserved_keyword(yytext); } +"private" { thrift_reserved_keyword(yytext); } +"protected" { thrift_reserved_keyword(yytext); } +"raise" { thrift_reserved_keyword(yytext); } +"return" { thrift_reserved_keyword(yytext); } +"sizeof" { thrift_reserved_keyword(yytext); } +"static" { thrift_reserved_keyword(yytext); } +"switch" { thrift_reserved_keyword(yytext); } +"synchronized" { thrift_reserved_keyword(yytext); } +"this" { thrift_reserved_keyword(yytext); } +"throw" { thrift_reserved_keyword(yytext); } +"transient" { thrift_reserved_keyword(yytext); } +"true" { thrift_reserved_keyword(yytext); } +"try" { thrift_reserved_keyword(yytext); } +"unsigned" { thrift_reserved_keyword(yytext); } +"var" { thrift_reserved_keyword(yytext); } +"virtual" { thrift_reserved_keyword(yytext); } +"volatile" { thrift_reserved_keyword(yytext); } +"while" { thrift_reserved_keyword(yytext); } +"with" { thrift_reserved_keyword(yytext); } +"union" { thrift_reserved_keyword(yytext); } +"yield" { thrift_reserved_keyword(yytext); } + +{intconstant} { + errno = 0; + yylval.iconst = strtoll(yytext, NULL, 10); + if (errno == ERANGE) { + integer_overflow(yytext); + } + return tok_int_constant; +} + +{hexconstant} { + errno = 0; + yylval.iconst = strtoll(yytext+2, NULL, 16); + if (errno == ERANGE) { + integer_overflow(yytext); + } + return tok_int_constant; +} + +{dubconstant} { + yylval.dconst = atof(yytext); + return tok_dub_constant; +} + +{identifier} { + yylval.id = strdup(yytext); + return tok_identifier; +} + +{st_identifier} { + yylval.id = strdup(yytext); + return tok_st_identifier; +} + +{literal_begin} { + char mark = yytext[0]; + std::string result; + for(;;) + { + int ch = yyinput(); + switch (ch) { + case EOF: + yyerror("End of file while read string at %d\n", yylineno); + exit(1); + case '\n': + yyerror("End of line while read string at %d\n", yylineno - 1); + exit(1); + case '\\': + ch = yyinput(); + switch (ch) { + case 'r': + result.push_back('\r'); + continue; + case 'n': + result.push_back('\n'); + continue; + case 't': + result.push_back('\t'); + continue; + case '"': + result.push_back('"'); + continue; + case '\'': + result.push_back('\''); + continue; + case '\\': + result.push_back('\\'); + continue; + default: + yyerror("Bad escape character\n"); + return -1; + } + break; + default: + if (ch == mark) { + yylval.id = strdup(result.c_str()); + return tok_literal; + } else { + result.push_back(ch); + } + } + } +} + + +{doctext} { + /* This does not show up in the parse tree. */ + /* Rather, the parser will grab it out of the global. */ + if (g_parse_mode == PROGRAM) { + clear_doctext(); + g_doctext = strdup(yytext + 3); + g_doctext[strlen(g_doctext) - 2] = '\0'; + g_doctext = clean_up_doctext(g_doctext); + g_doctext_lineno = yylineno; + } +} + + +%% + +/* vim: filetype=lex +*/ diff --git a/compiler/cpp/src/thrifty.yy b/compiler/cpp/src/thrifty.yy new file mode 100644 index 00000000..bf5408e3 --- /dev/null +++ b/compiler/cpp/src/thrifty.yy @@ -0,0 +1,1127 @@ +%{ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Thrift parser. + * + * This parser is used on a thrift definition file. + * + */ + +#define __STDC_LIMIT_MACROS +#define __STDC_FORMAT_MACROS +#include +#include +#include +#include "main.h" +#include "globals.h" +#include "parse/t_program.h" +#include "parse/t_scope.h" + +/** + * This global variable is used for automatic numbering of field indices etc. + * when parsing the members of a struct. Field values are automatically + * assigned starting from -1 and working their way down. + */ +int y_field_val = -1; +int g_arglist = 0; + +%} + +/** + * This structure is used by the parser to hold the data types associated with + * various parse nodes. + */ +%union { + char* id; + int64_t iconst; + double dconst; + bool tbool; + t_doc* tdoc; + t_type* ttype; + t_base_type* tbase; + t_typedef* ttypedef; + t_enum* tenum; + t_enum_value* tenumv; + t_const* tconst; + t_const_value* tconstv; + t_struct* tstruct; + t_service* tservice; + t_function* tfunction; + t_field* tfield; + char* dtext; + t_field::e_req ereq; + t_annotation* tannot; +} + +/** + * Strings identifier + */ +%token tok_identifier +%token tok_literal +%token tok_doctext +%token tok_st_identifier + +/** + * Constant values + */ +%token tok_int_constant +%token tok_dub_constant + +/** + * Header keywords + */ +%token tok_include +%token tok_namespace +%token tok_cpp_namespace +%token tok_cpp_include +%token tok_cpp_type +%token tok_php_namespace +%token tok_py_module +%token tok_perl_package +%token tok_java_package +%token tok_xsd_all +%token tok_xsd_optional +%token tok_xsd_nillable +%token tok_xsd_namespace +%token tok_xsd_attrs +%token tok_ruby_namespace +%token tok_smalltalk_category +%token tok_smalltalk_prefix +%token tok_cocoa_prefix +%token tok_csharp_namespace + +/** + * Base datatype keywords + */ +%token tok_void +%token tok_bool +%token tok_byte +%token tok_string +%token tok_binary +%token tok_slist +%token tok_senum +%token tok_i16 +%token tok_i32 +%token tok_i64 +%token tok_double + +/** + * Complex type keywords + */ +%token tok_map +%token tok_list +%token tok_set + +/** + * Function modifiers + */ +%token tok_oneway + +/** + * Thrift language keywords + */ +%token tok_typedef +%token tok_struct +%token tok_xception +%token tok_throws +%token tok_extends +%token tok_service +%token tok_enum +%token tok_const +%token tok_required +%token tok_optional + +/** + * Grammar nodes + */ + +%type BaseType +%type ContainerType +%type SimpleContainerType +%type MapType +%type SetType +%type ListType + +%type Definition +%type TypeDefinition + +%type Typedef +%type DefinitionType + +%type TypeAnnotations +%type TypeAnnotationList +%type TypeAnnotation + +%type Field +%type FieldIdentifier +%type FieldRequiredness +%type FieldType +%type FieldValue +%type FieldList + +%type Enum +%type EnumDefList +%type EnumDef + +%type Senum +%type SenumDefList +%type SenumDef + +%type Const +%type ConstValue +%type ConstList +%type ConstListContents +%type ConstMap +%type ConstMapContents + +%type Struct +%type Xception +%type Service + +%type Function +%type FunctionType +%type FunctionList + +%type Throws +%type Extends +%type Oneway +%type XsdAll +%type XsdOptional +%type XsdNillable +%type XsdAttributes +%type CppType + +%type CaptureDocText + +%% + +/** + * Thrift Grammar Implementation. + * + * For the most part this source file works its way top down from what you + * might expect to find in a typical .thrift file, i.e. type definitions and + * namespaces up top followed by service definitions using those types. + */ + +Program: + HeaderList DefinitionList + { + pdebug("Program -> Headers DefinitionList"); + /* + TODO(dreiss): Decide whether full-program doctext is worth the trouble. + if ($1 != NULL) { + g_program->set_doc($1); + } + */ + clear_doctext(); + } + +CaptureDocText: + { + if (g_parse_mode == PROGRAM) { + $$ = g_doctext; + g_doctext = NULL; + } else { + $$ = NULL; + } + } + +/* TODO(dreiss): Try to DestroyDocText in all sorts or random places. */ +DestroyDocText: + { + if (g_parse_mode == PROGRAM) { + clear_doctext(); + } + } + +/* We have to DestroyDocText here, otherwise it catches the doctext + on the first real element. */ +HeaderList: + HeaderList DestroyDocText Header + { + pdebug("HeaderList -> HeaderList Header"); + } +| + { + pdebug("HeaderList -> "); + } + +Header: + Include + { + pdebug("Header -> Include"); + } +| tok_namespace tok_identifier tok_identifier + { + pdebug("Header -> tok_namespace tok_identifier tok_identifier"); + if (g_parse_mode == PROGRAM) { + g_program->set_namespace($2, $3); + } + } +/* TODO(dreiss): Get rid of this once everyone is using the new hotness. */ +| tok_cpp_namespace tok_identifier + { + pwarning(1, "'cpp_namespace' is deprecated. Use 'namespace cpp' instead"); + pdebug("Header -> tok_cpp_namespace tok_identifier"); + if (g_parse_mode == PROGRAM) { + g_program->set_namespace("cpp", $2); + } + } +| tok_cpp_include tok_literal + { + pdebug("Header -> tok_cpp_include tok_literal"); + if (g_parse_mode == PROGRAM) { + g_program->add_cpp_include($2); + } + } +| tok_php_namespace tok_identifier + { + pwarning(1, "'php_namespace' is deprecated. Use 'namespace php' instead"); + pdebug("Header -> tok_php_namespace tok_identifier"); + if (g_parse_mode == PROGRAM) { + g_program->set_namespace("php", $2); + } + } +/* TODO(dreiss): Get rid of this once everyone is using the new hotness. */ +| tok_py_module tok_identifier + { + pwarning(1, "'py_module' is deprecated. Use 'namespace py' instead"); + pdebug("Header -> tok_py_module tok_identifier"); + if (g_parse_mode == PROGRAM) { + g_program->set_namespace("py", $2); + } + } +/* TODO(dreiss): Get rid of this once everyone is using the new hotness. */ +| tok_perl_package tok_identifier + { + pwarning(1, "'perl_package' is deprecated. Use 'namespace perl' instead"); + pdebug("Header -> tok_perl_namespace tok_identifier"); + if (g_parse_mode == PROGRAM) { + g_program->set_namespace("perl", $2); + } + } +/* TODO(dreiss): Get rid of this once everyone is using the new hotness. */ +| tok_ruby_namespace tok_identifier + { + pwarning(1, "'ruby_namespace' is deprecated. Use 'namespace rb' instead"); + pdebug("Header -> tok_ruby_namespace tok_identifier"); + if (g_parse_mode == PROGRAM) { + g_program->set_namespace("rb", $2); + } + } +/* TODO(dreiss): Get rid of this once everyone is using the new hotness. */ +| tok_smalltalk_category tok_st_identifier + { + pwarning(1, "'smalltalk_category' is deprecated. Use 'namespace smalltalk.category' instead"); + pdebug("Header -> tok_smalltalk_category tok_st_identifier"); + if (g_parse_mode == PROGRAM) { + g_program->set_namespace("smalltalk.category", $2); + } + } +/* TODO(dreiss): Get rid of this once everyone is using the new hotness. */ +| tok_smalltalk_prefix tok_identifier + { + pwarning(1, "'smalltalk_prefix' is deprecated. Use 'namespace smalltalk.prefix' instead"); + pdebug("Header -> tok_smalltalk_prefix tok_identifier"); + if (g_parse_mode == PROGRAM) { + g_program->set_namespace("smalltalk.prefix", $2); + } + } +/* TODO(dreiss): Get rid of this once everyone is using the new hotness. */ +| tok_java_package tok_identifier + { + pwarning(1, "'java_package' is deprecated. Use 'namespace java' instead"); + pdebug("Header -> tok_java_package tok_identifier"); + if (g_parse_mode == PROGRAM) { + g_program->set_namespace("java", $2); + } + } +/* TODO(dreiss): Get rid of this once everyone is using the new hotness. */ +| tok_cocoa_prefix tok_identifier + { + pwarning(1, "'cocoa_prefix' is deprecated. Use 'namespace cocoa' instead"); + pdebug("Header -> tok_cocoa_prefix tok_identifier"); + if (g_parse_mode == PROGRAM) { + g_program->set_namespace("cocoa", $2); + } + } +/* TODO(dreiss): Get rid of this once everyone is using the new hotness. */ +| tok_xsd_namespace tok_literal + { + pwarning(1, "'xsd_namespace' is deprecated. Use 'namespace xsd' instead"); + pdebug("Header -> tok_xsd_namespace tok_literal"); + if (g_parse_mode == PROGRAM) { + g_program->set_namespace("cocoa", $2); + } + } +/* TODO(dreiss): Get rid of this once everyone is using the new hotness. */ +| tok_csharp_namespace tok_identifier + { + pwarning(1, "'csharp_namespace' is deprecated. Use 'namespace csharp' instead"); + pdebug("Header -> tok_csharp_namespace tok_identifier"); + if (g_parse_mode == PROGRAM) { + g_program->set_namespace("csharp", $2); + } + } + +Include: + tok_include tok_literal + { + pdebug("Include -> tok_include tok_literal"); + if (g_parse_mode == INCLUDES) { + std::string path = include_file(std::string($2)); + if (!path.empty()) { + g_program->add_include(path, std::string($2)); + } + } + } + +DefinitionList: + DefinitionList CaptureDocText Definition + { + pdebug("DefinitionList -> DefinitionList Definition"); + if ($2 != NULL && $3 != NULL) { + $3->set_doc($2); + } + } +| + { + pdebug("DefinitionList -> "); + } + +Definition: + Const + { + pdebug("Definition -> Const"); + if (g_parse_mode == PROGRAM) { + g_program->add_const($1); + } + $$ = $1; + } +| TypeDefinition + { + pdebug("Definition -> TypeDefinition"); + if (g_parse_mode == PROGRAM) { + g_scope->add_type($1->get_name(), $1); + if (g_parent_scope != NULL) { + g_parent_scope->add_type(g_parent_prefix + $1->get_name(), $1); + } + } + $$ = $1; + } +| Service + { + pdebug("Definition -> Service"); + if (g_parse_mode == PROGRAM) { + g_scope->add_service($1->get_name(), $1); + if (g_parent_scope != NULL) { + g_parent_scope->add_service(g_parent_prefix + $1->get_name(), $1); + } + g_program->add_service($1); + } + $$ = $1; + } + +TypeDefinition: + Typedef + { + pdebug("TypeDefinition -> Typedef"); + if (g_parse_mode == PROGRAM) { + g_program->add_typedef($1); + } + } +| Enum + { + pdebug("TypeDefinition -> Enum"); + if (g_parse_mode == PROGRAM) { + g_program->add_enum($1); + } + } +| Senum + { + pdebug("TypeDefinition -> Senum"); + if (g_parse_mode == PROGRAM) { + g_program->add_typedef($1); + } + } +| Struct + { + pdebug("TypeDefinition -> Struct"); + if (g_parse_mode == PROGRAM) { + g_program->add_struct($1); + } + } +| Xception + { + pdebug("TypeDefinition -> Xception"); + if (g_parse_mode == PROGRAM) { + g_program->add_xception($1); + } + } + +Typedef: + tok_typedef DefinitionType tok_identifier + { + pdebug("TypeDef -> tok_typedef DefinitionType tok_identifier"); + t_typedef *td = new t_typedef(g_program, $2, $3); + $$ = td; + } + +CommaOrSemicolonOptional: + ',' + {} +| ';' + {} +| + {} + +Enum: + tok_enum tok_identifier '{' EnumDefList '}' + { + pdebug("Enum -> tok_enum tok_identifier { EnumDefList }"); + $$ = $4; + $$->set_name($2); + } + +EnumDefList: + EnumDefList EnumDef + { + pdebug("EnumDefList -> EnumDefList EnumDef"); + $$ = $1; + $$->append($2); + } +| + { + pdebug("EnumDefList -> "); + $$ = new t_enum(g_program); + } + +EnumDef: + CaptureDocText tok_identifier '=' tok_int_constant CommaOrSemicolonOptional + { + pdebug("EnumDef -> tok_identifier = tok_int_constant"); + if ($4 < 0) { + pwarning(1, "Negative value supplied for enum %s.\n", $2); + } + if ($4 > INT_MAX) { + pwarning(1, "64-bit value supplied for enum %s.\n", $2); + } + $$ = new t_enum_value($2, $4); + if ($1 != NULL) { + $$->set_doc($1); + } + if (g_parse_mode == PROGRAM) { + g_scope->add_constant($2, new t_const(g_type_i32, $2, new t_const_value($4))); + if (g_parent_scope != NULL) { + g_parent_scope->add_constant(g_parent_prefix + $2, new t_const(g_type_i32, $2, new t_const_value($4))); + } + } + } +| + CaptureDocText tok_identifier CommaOrSemicolonOptional + { + pdebug("EnumDef -> tok_identifier"); + $$ = new t_enum_value($2); + if ($1 != NULL) { + $$->set_doc($1); + } + } + +Senum: + tok_senum tok_identifier '{' SenumDefList '}' + { + pdebug("Senum -> tok_senum tok_identifier { SenumDefList }"); + $$ = new t_typedef(g_program, $4, $2); + } + +SenumDefList: + SenumDefList SenumDef + { + pdebug("SenumDefList -> SenumDefList SenumDef"); + $$ = $1; + $$->add_string_enum_val($2); + } +| + { + pdebug("SenumDefList -> "); + $$ = new t_base_type("string", t_base_type::TYPE_STRING); + $$->set_string_enum(true); + } + +SenumDef: + tok_literal CommaOrSemicolonOptional + { + pdebug("SenumDef -> tok_literal"); + $$ = $1; + } + +Const: + tok_const FieldType tok_identifier '=' ConstValue CommaOrSemicolonOptional + { + pdebug("Const -> tok_const FieldType tok_identifier = ConstValue"); + if (g_parse_mode == PROGRAM) { + $$ = new t_const($2, $3, $5); + validate_const_type($$); + + g_scope->add_constant($3, $$); + if (g_parent_scope != NULL) { + g_parent_scope->add_constant(g_parent_prefix + $3, $$); + } + + } else { + $$ = NULL; + } + } + +ConstValue: + tok_int_constant + { + pdebug("ConstValue => tok_int_constant"); + $$ = new t_const_value(); + $$->set_integer($1); + if ($1 < INT32_MIN || $1 > INT32_MAX) { + pwarning(1, "64-bit constant \"%"PRIi64"\" may not work in all languages.\n", $1); + } + } +| tok_dub_constant + { + pdebug("ConstValue => tok_dub_constant"); + $$ = new t_const_value(); + $$->set_double($1); + } +| tok_literal + { + pdebug("ConstValue => tok_literal"); + $$ = new t_const_value($1); + } +| tok_identifier + { + pdebug("ConstValue => tok_identifier"); + t_const* constant = g_scope->get_constant($1); + if (constant != NULL) { + $$ = constant->get_value(); + } else { + if (g_parse_mode == PROGRAM) { + pwarning(1, "Constant strings should be quoted: %s\n", $1); + } + $$ = new t_const_value($1); + } + } +| ConstList + { + pdebug("ConstValue => ConstList"); + $$ = $1; + } +| ConstMap + { + pdebug("ConstValue => ConstMap"); + $$ = $1; + } + +ConstList: + '[' ConstListContents ']' + { + pdebug("ConstList => [ ConstListContents ]"); + $$ = $2; + } + +ConstListContents: + ConstListContents ConstValue CommaOrSemicolonOptional + { + pdebug("ConstListContents => ConstListContents ConstValue CommaOrSemicolonOptional"); + $$ = $1; + $$->add_list($2); + } +| + { + pdebug("ConstListContents =>"); + $$ = new t_const_value(); + $$->set_list(); + } + +ConstMap: + '{' ConstMapContents '}' + { + pdebug("ConstMap => { ConstMapContents }"); + $$ = $2; + } + +ConstMapContents: + ConstMapContents ConstValue ':' ConstValue CommaOrSemicolonOptional + { + pdebug("ConstMapContents => ConstMapContents ConstValue CommaOrSemicolonOptional"); + $$ = $1; + $$->add_map($2, $4); + } +| + { + pdebug("ConstMapContents =>"); + $$ = new t_const_value(); + $$->set_map(); + } + +Struct: + tok_struct tok_identifier XsdAll '{' FieldList '}' TypeAnnotations + { + pdebug("Struct -> tok_struct tok_identifier { FieldList }"); + $5->set_xsd_all($3); + $$ = $5; + $$->set_name($2); + if ($7 != NULL) { + $$->annotations_ = $7->annotations_; + delete $7; + } + } + +XsdAll: + tok_xsd_all + { + $$ = true; + } +| + { + $$ = false; + } + +XsdOptional: + tok_xsd_optional + { + $$ = true; + } +| + { + $$ = false; + } + +XsdNillable: + tok_xsd_nillable + { + $$ = true; + } +| + { + $$ = false; + } + +XsdAttributes: + tok_xsd_attrs '{' FieldList '}' + { + $$ = $3; + } +| + { + $$ = NULL; + } + +Xception: + tok_xception tok_identifier '{' FieldList '}' + { + pdebug("Xception -> tok_xception tok_identifier { FieldList }"); + $4->set_name($2); + $4->set_xception(true); + $$ = $4; + } + +Service: + tok_service tok_identifier Extends '{' FlagArgs FunctionList UnflagArgs '}' + { + pdebug("Service -> tok_service tok_identifier { FunctionList }"); + $$ = $6; + $$->set_name($2); + $$->set_extends($3); + } + +FlagArgs: + { + g_arglist = 1; + } + +UnflagArgs: + { + g_arglist = 0; + } + +Extends: + tok_extends tok_identifier + { + pdebug("Extends -> tok_extends tok_identifier"); + $$ = NULL; + if (g_parse_mode == PROGRAM) { + $$ = g_scope->get_service($2); + if ($$ == NULL) { + yyerror("Service \"%s\" has not been defined.", $2); + exit(1); + } + } + } +| + { + $$ = NULL; + } + +FunctionList: + FunctionList Function + { + pdebug("FunctionList -> FunctionList Function"); + $$ = $1; + $1->add_function($2); + } +| + { + pdebug("FunctionList -> "); + $$ = new t_service(g_program); + } + +Function: + CaptureDocText Oneway FunctionType tok_identifier '(' FieldList ')' Throws CommaOrSemicolonOptional + { + $6->set_name(std::string($4) + "_args"); + $$ = new t_function($3, $4, $6, $8, $2); + if ($1 != NULL) { + $$->set_doc($1); + } + } + +Oneway: + tok_oneway + { + $$ = true; + } +| + { + $$ = false; + } + +Throws: + tok_throws '(' FieldList ')' + { + pdebug("Throws -> tok_throws ( FieldList )"); + $$ = $3; + if (g_parse_mode == PROGRAM && !validate_throws($$)) { + yyerror("Throws clause may not contain non-exception types"); + exit(1); + } + } +| + { + $$ = new t_struct(g_program); + } + +FieldList: + FieldList Field + { + pdebug("FieldList -> FieldList , Field"); + $$ = $1; + if (!($$->append($2))) { + yyerror("Field identifier %d for \"%s\" has already been used", $2->get_key(), $2->get_name().c_str()); + exit(1); + } + } +| + { + pdebug("FieldList -> "); + y_field_val = -1; + $$ = new t_struct(g_program); + } + +Field: + CaptureDocText FieldIdentifier FieldRequiredness FieldType tok_identifier FieldValue XsdOptional XsdNillable XsdAttributes CommaOrSemicolonOptional + { + pdebug("tok_int_constant : Field -> FieldType tok_identifier"); + if ($2 < 0) { + pwarning(1, "No field key specified for %s, resulting protocol may have conflicts or not be backwards compatible!\n", $5); + if (g_strict >= 192) { + yyerror("Implicit field keys are deprecated and not allowed with -strict"); + exit(1); + } + } + $$ = new t_field($4, $5, $2); + $$->set_req($3); + if ($6 != NULL) { + validate_field_value($$, $6); + $$->set_value($6); + } + $$->set_xsd_optional($7); + $$->set_xsd_nillable($8); + if ($1 != NULL) { + $$->set_doc($1); + } + if ($9 != NULL) { + $$->set_xsd_attrs($9); + } + } + +FieldIdentifier: + tok_int_constant ':' + { + if ($1 <= 0) { + pwarning(1, "Nonpositive value (%d) not allowed as a field key.\n", $1); + $1 = y_field_val--; + } + $$ = $1; + } +| + { + $$ = y_field_val--; + } + +FieldRequiredness: + tok_required + { + if (g_arglist) { + if (g_parse_mode == PROGRAM) { + pwarning(1, "required keyword is ignored in argument lists.\n"); + } + $$ = t_field::T_OPT_IN_REQ_OUT; + } else { + $$ = t_field::T_REQUIRED; + } + } +| tok_optional + { + if (g_arglist) { + if (g_parse_mode == PROGRAM) { + pwarning(1, "optional keyword is ignored in argument lists.\n"); + } + $$ = t_field::T_OPT_IN_REQ_OUT; + } else { + $$ = t_field::T_OPTIONAL; + } + } +| + { + $$ = t_field::T_OPT_IN_REQ_OUT; + } + +FieldValue: + '=' ConstValue + { + if (g_parse_mode == PROGRAM) { + $$ = $2; + } else { + $$ = NULL; + } + } +| + { + $$ = NULL; + } + +DefinitionType: + BaseType + { + pdebug("DefinitionType -> BaseType"); + $$ = $1; + } +| ContainerType + { + pdebug("DefinitionType -> ContainerType"); + $$ = $1; + } + +FunctionType: + FieldType + { + pdebug("FunctionType -> FieldType"); + $$ = $1; + } +| tok_void + { + pdebug("FunctionType -> tok_void"); + $$ = g_type_void; + } + +FieldType: + tok_identifier + { + pdebug("FieldType -> tok_identifier"); + if (g_parse_mode == INCLUDES) { + // Ignore identifiers in include mode + $$ = NULL; + } else { + // Lookup the identifier in the current scope + $$ = g_scope->get_type($1); + if ($$ == NULL) { + yyerror("Type \"%s\" has not been defined.", $1); + exit(1); + } + } + } +| BaseType + { + pdebug("FieldType -> BaseType"); + $$ = $1; + } +| ContainerType + { + pdebug("FieldType -> ContainerType"); + $$ = $1; + } + +BaseType: + tok_string + { + pdebug("BaseType -> tok_string"); + $$ = g_type_string; + } +| tok_binary + { + pdebug("BaseType -> tok_binary"); + $$ = g_type_binary; + } +| tok_slist + { + pdebug("BaseType -> tok_slist"); + $$ = g_type_slist; + } +| tok_bool + { + pdebug("BaseType -> tok_bool"); + $$ = g_type_bool; + } +| tok_byte + { + pdebug("BaseType -> tok_byte"); + $$ = g_type_byte; + } +| tok_i16 + { + pdebug("BaseType -> tok_i16"); + $$ = g_type_i16; + } +| tok_i32 + { + pdebug("BaseType -> tok_i32"); + $$ = g_type_i32; + } +| tok_i64 + { + pdebug("BaseType -> tok_i64"); + $$ = g_type_i64; + } +| tok_double + { + pdebug("BaseType -> tok_double"); + $$ = g_type_double; + } + +ContainerType: SimpleContainerType TypeAnnotations + { + pdebug("ContainerType -> SimpleContainerType TypeAnnotations"); + $$ = $1; + if ($2 != NULL) { + $$->annotations_ = $2->annotations_; + delete $2; + } + } + +SimpleContainerType: + MapType + { + pdebug("SimpleContainerType -> MapType"); + $$ = $1; + } +| SetType + { + pdebug("SimpleContainerType -> SetType"); + $$ = $1; + } +| ListType + { + pdebug("SimpleContainerType -> ListType"); + $$ = $1; + } + +MapType: + tok_map CppType '<' FieldType ',' FieldType '>' + { + pdebug("MapType -> tok_map "); + $$ = new t_map($4, $6); + if ($2 != NULL) { + ((t_container*)$$)->set_cpp_name(std::string($2)); + } + } + +SetType: + tok_set CppType '<' FieldType '>' + { + pdebug("SetType -> tok_set"); + $$ = new t_set($4); + if ($2 != NULL) { + ((t_container*)$$)->set_cpp_name(std::string($2)); + } + } + +ListType: + tok_list '<' FieldType '>' CppType + { + pdebug("ListType -> tok_list"); + $$ = new t_list($3); + if ($5 != NULL) { + ((t_container*)$$)->set_cpp_name(std::string($5)); + } + } + +CppType: + tok_cpp_type tok_literal + { + $$ = $2; + } +| + { + $$ = NULL; + } + +TypeAnnotations: + '(' TypeAnnotationList ')' + { + pdebug("TypeAnnotations -> ( TypeAnnotationList )"); + $$ = $2; + } +| + { + $$ = NULL; + } + +TypeAnnotationList: + TypeAnnotationList TypeAnnotation + { + pdebug("TypeAnnotationList -> TypeAnnotationList , TypeAnnotation"); + $$ = $1; + $$->annotations_[$2->key] = $2->val; + delete $2; + } +| + { + /* Just use a dummy structure to hold the annotations. */ + $$ = new t_struct(g_program); + } + +TypeAnnotation: + tok_identifier '=' tok_literal CommaOrSemicolonOptional + { + pdebug("TypeAnnotation -> tok_identifier = tok_literal"); + $$ = new t_annotation; + $$->key = $1; + $$->val = $3; + } + +%% diff --git a/configure.ac b/configure.ac new file mode 100644 index 00000000..6bba72d1 --- /dev/null +++ b/configure.ac @@ -0,0 +1,226 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +AC_PREREQ(2.59) + +AC_INIT([thrift], [0.1-dev]) + +AC_CONFIG_AUX_DIR([.]) + +AM_INIT_AUTOMAKE + +AC_ARG_VAR([PY_PREFIX], [Prefix for installing Python modules. + (Normal --prefix is ignored for Python because + Python has different conventions.) + Default = "/usr"]) +AS_IF([test "x$PY_PREFIX" = x], [PY_PREFIX="/usr"]) + +AC_ARG_VAR([JAVA_PREFIX], [Prefix for installing the Java lib jar. + (Normal --prefix is ignored for Java because + Java has different conevntions.) + Default = "/usr/local/lib"]) +AS_IF([test "x$JAVA_PREFIX" = x], [JAVA_PREFIX="/usr/local/lib"]) + +AC_PROG_CC +AC_PROG_CPP +AC_PROG_CXX +AC_PROG_INSTALL +AC_PROG_LIBTOOL +AC_PROG_MAKE_SET +AC_PROG_YACC +AC_PROG_LEX +AM_PROG_LEX +AC_PROG_LN_S +AC_PROG_MKDIR_P + +AC_LANG([C++]) +AX_BOOST_BASE([1.33.1]) + +AX_LIB_EVENT([1.0]) +AM_CONDITIONAL([AMX_HAVE_LIBEVENT], [test "$success" = "yes"]) + +AX_LIB_ZLIB([1.2.3]) +AM_CONDITIONAL([AMX_HAVE_ZLIB], [test "$success" = "yes"]) + +AX_THRIFT_LIB(csharp, [C#], yes) +if test "$with_csharp" = "yes"; then + PKG_CHECK_MODULES(MONO, mono >= 2.0.0, net_3_5=yes, net_3_5=no) + PKG_CHECK_MODULES(MONO, mono >= 1.2.4, have_mono=yes, have_mono=no) +fi +AM_CONDITIONAL(WITH_MONO, [test "$have_mono" = "yes"]) +AM_CONDITIONAL(NET_2_0, [test "$net_3_5" = "no"]) + +AX_THRIFT_LIB(java, [Java], yes) +if test "$with_java" = "yes"; then + AX_JAVAC_AND_JAVA + AC_PATH_PROG([ANT], [ant]) +fi +AM_CONDITIONAL([WITH_JAVA], + [test -n "$ANT" -a -n "$JAVA" -a -n "$JAVAC"]) + +AX_THRIFT_LIB(erlang, [Erlang], yes) +if test "$with_erlang" = "yes"; then + AC_PATH_PROG([ERLC], [erlc]) +fi +AM_CONDITIONAL(WITH_ERLANG, [test -n "$ERLC"]) + +AX_THRIFT_LIB(py, [Python], yes) +if test "$with_py" = "yes"; then + AM_PATH_PYTHON(2.4,, :) +fi +AM_CONDITIONAL(WITH_PYTHON, [test -n "$PYTHON" -a "$PYTHON" != ":"]) + +AX_THRIFT_LIB(perl, [Perl], yes) +if test "$with_perl" = "yes"; then + AC_PATH_PROG([PERL], [perl]) +fi +AM_CONDITIONAL(WITH_PERL, [test -n "$PERL"]) + +AX_THRIFT_LIB(ruby, [Ruby], yes) +if test "$with_ruby" = "yes"; then + AC_PATH_PROG([RUBY], [ruby]) + AC_PATH_PROG([RSPEC], [spec]) +fi +AM_CONDITIONAL(WITH_RUBY, [test -n "$RUBY"]) +AM_CONDITIONAL(HAVE_RSPEC, [test -n "$RSPEC"]) + + +AC_C_CONST +AC_C_INLINE +AC_C_VOLATILE + +AC_HEADER_STDBOOL +AC_HEADER_STDC +AC_HEADER_TIME +AC_CHECK_HEADERS([arpa/inet.h]) +AC_CHECK_HEADERS([endian.h]) +AC_CHECK_HEADERS([fcntl.h]) +AC_CHECK_HEADERS([inttypes.h]) +AC_CHECK_HEADERS([limits.h]) +AC_CHECK_HEADERS([netdb.h]) +AC_CHECK_HEADERS([netinet/in.h]) +AC_CHECK_HEADERS([pthread.h]) +AC_CHECK_HEADERS([stddef.h]) +AC_CHECK_HEADERS([stdlib.h]) +AC_CHECK_HEADERS([sys/socket.h]) +AC_CHECK_HEADERS([sys/time.h]) +AC_CHECK_HEADERS([unistd.h]) +AC_CHECK_HEADERS([libintl.h]) +AC_CHECK_HEADERS([malloc.h]) + +AC_CHECK_LIB(pthread, pthread_create) +AC_CHECK_LIB(rt, sched_get_priority_min) + +AC_TYPE_INT16_T +AC_TYPE_INT32_T +AC_TYPE_INT64_T +AC_TYPE_INT8_T +AC_TYPE_MODE_T +AC_TYPE_OFF_T +AC_TYPE_SIZE_T +AC_TYPE_SSIZE_T +AC_TYPE_UINT16_T +AC_TYPE_UINT32_T +AC_TYPE_UINT64_T +AC_TYPE_UINT8_T +AC_CHECK_TYPES([ptrdiff_t], [], [exit 1]) + +AC_STRUCT_TM + +AC_FUNC_ALLOCA +AC_FUNC_MALLOC +AC_FUNC_MEMCMP +AC_FUNC_REALLOC +AC_FUNC_SELECT_ARGTYPES +AC_FUNC_STAT +AC_FUNC_STRERROR_R +AC_FUNC_STRFTIME +AC_FUNC_VPRINTF +AC_CHECK_FUNCS([strtoul]) +AC_CHECK_FUNCS([bzero]) +AC_CHECK_FUNCS([clock_gettime]) +AC_CHECK_FUNCS([ftruncate]) +AC_CHECK_FUNCS([gethostbyname]) +AC_CHECK_FUNCS([gettimeofday]) +AC_CHECK_FUNCS([memmove]) +AC_CHECK_FUNCS([memset]) +AC_CHECK_FUNCS([mkdir]) +AC_CHECK_FUNCS([realpath]) +AC_CHECK_FUNCS([select]) +AC_CHECK_FUNCS([socket]) +AC_CHECK_FUNCS([strchr]) +AC_CHECK_FUNCS([strdup]) +AC_CHECK_FUNCS([strerror]) +AC_CHECK_FUNCS([strstr]) +AC_CHECK_FUNCS([strtol]) +AC_CHECK_FUNCS([sqrt]) + +AX_SIGNED_RIGHT_SHIFT + +AX_THRIFT_GEN(cpp, [C++], yes) +AM_CONDITIONAL([THRIFT_GEN_cpp], [test "$ax_thrift_gen_cpp" = "yes"]) +AX_THRIFT_GEN(java, [Java], yes) +AM_CONDITIONAL([THRIFT_GEN_java], [test "$ax_thrift_gen_java" = "yes"]) +AX_THRIFT_GEN(csharp, [C#], yes) +AM_CONDITIONAL([THRIFT_GEN_csharp], [test "$ax_thrift_gen_csharp" = "yes"]) +AX_THRIFT_GEN(py, [Python], yes) +AM_CONDITIONAL([THRIFT_GEN_py], [test "$ax_thrift_gen_py" = "yes"]) +AX_THRIFT_GEN(rb, [Ruby], yes) +AM_CONDITIONAL([THRIFT_GEN_rb], [test "$ax_thrift_gen_rb" = "yes"]) +AX_THRIFT_GEN(perl, [Perl], yes) +AM_CONDITIONAL([THRIFT_GEN_perl], [test "$ax_thrift_gen_perl" = "yes"]) +AX_THRIFT_GEN(php, [PHP], yes) +AM_CONDITIONAL([THRIFT_GEN_php], [test "$ax_thrift_gen_php" = "yes"]) +AX_THRIFT_GEN(erl, [Erlang], yes) +AM_CONDITIONAL([THRIFT_GEN_erl], [test "$ax_thrift_gen_erl" = "yes"]) +AX_THRIFT_GEN(cocoa, [Cocoa], yes) +AM_CONDITIONAL([THRIFT_GEN_cocoa], [test "$ax_thrift_gen_cocoa" = "yes"]) +AX_THRIFT_GEN(st, [Smalltalk], yes) +AM_CONDITIONAL([THRIFT_GEN_st], [test "$ax_thrift_gen_st" = "yes"]) +AX_THRIFT_GEN(ocaml, [OCaml], yes) +AM_CONDITIONAL([THRIFT_GEN_ocaml], [test "$ax_thrift_gen_ocaml" = "yes"]) +AX_THRIFT_GEN(hs, [Haskell], yes) +AM_CONDITIONAL([THRIFT_GEN_hs], [test "$ax_thrift_gen_hs" = "yes"]) +AX_THRIFT_GEN(xsd, [XSD], yes) +AM_CONDITIONAL([THRIFT_GEN_xsd], [test "$ax_thrift_gen_xsd" = "yes"]) +AX_THRIFT_GEN(html, [HTML], yes) +AM_CONDITIONAL([THRIFT_GEN_html], [test "$ax_thrift_gen_html" = "yes"]) + +AC_CONFIG_HEADERS(config.h:config.hin) + +AC_CONFIG_FILES([ + Makefile + compiler/cpp/Makefile + lib/Makefile + lib/cpp/Makefile + lib/cpp/thrift.pc + lib/cpp/thrift-nb.pc + lib/cpp/thrift-z.pc + lib/csharp/Makefile + lib/java/Makefile + lib/perl/Makefile + lib/perl/test/Makefile + lib/py/Makefile + lib/rb/Makefile + test/Makefile + test/py/Makefile + test/rb/Makefile +]) + +AC_OUTPUT diff --git a/contrib/fb303/LICENSE b/contrib/fb303/LICENSE new file mode 100644 index 00000000..4eacb643 --- /dev/null +++ b/contrib/fb303/LICENSE @@ -0,0 +1,16 @@ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. diff --git a/contrib/fb303/Makefile.am b/contrib/fb303/Makefile.am new file mode 100644 index 00000000..de7fbb60 --- /dev/null +++ b/contrib/fb303/Makefile.am @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +@GLOBAL_HEADER_MK@ + +@PRODUCT_MK@ + +SUBDIRS = . cpp py + +BUILT_SOURCES = + + +clean-local: clean-common + +@GLOBAL_FOOTER_MK@ diff --git a/contrib/fb303/README b/contrib/fb303/README new file mode 100644 index 00000000..8ade560c --- /dev/null +++ b/contrib/fb303/README @@ -0,0 +1,37 @@ +Project FB303: The Facebook Bassline +------------------------------------ + +* Curious about the 303? * +http://en.wikipedia.org/wiki/Roland_TB-303 + +* Why the name? * +The TB303 makes bass lines. +.Bass is what lies underneath any strong tune. +..fb303 is the shared root of all thrift services. +...fb303 => FacebookBase303. + +* How do I use this? * +Take a look at the examples to see how your backend project can +and should inherit from this service. + +* What does it provide? * +A standard interface to monitoring, dynamic options and configuration, +uptime reports, activity, etc. + +* I want more. * +Think carefully first about whether the functionality you are going to add +belongs here or in your application. If it can be abstracted and is generally +useful, then it probably belongs somewhere in the fb303 tree. Keep in mind, +not every product has to use ALL the functionality of fb303, but every product +CANNOT use functionality that is NOT in fb303. + +* Is this open source? * +Yes. fb303 is distributed under the Thrift Software License. See the +LICENSE file for more details. + +* Installation * +fb303 is configured/built/installed similar to Thrift. See the README +in the Thrift root directory for more information. + +* Who wrote this README? * +mcslee@facebook.com diff --git a/contrib/fb303/acinclude.m4 b/contrib/fb303/acinclude.m4 new file mode 100644 index 00000000..faafba6b --- /dev/null +++ b/contrib/fb303/acinclude.m4 @@ -0,0 +1,258 @@ +dnl Copyright (C) 2009 Facebook +dnl Copying and distribution of this file, with or without modification, +dnl are permitted in any medium without royalty provided the copyright +dnl notice and this notice are preserved. + +AC_DEFUN([FB_INITIALIZE], +[ +AM_INIT_AUTOMAKE([ foreign 1.9.5 no-define ]) +if test "x$1" = "xlocalinstall"; then +wdir=`pwd` +# To use $wdir undef quote. +# +########## +AC_PREFIX_DEFAULT([`pwd`/install]) +echo +fi +AC_PROG_CC +AC_PROG_CXX +AC_PROG_RANLIB(RANLIB, ranlib) +AC_PATH_PROGS(BASH, bash) +AC_PATH_PROGS(PERL, perl) +AC_PATH_PROGS(PYTHON, python) +AC_PATH_PROGS(AR, ar) +AC_PATH_PROGS(ANT, ant) +PRODUCT_MK="" +]) + +AC_DEFUN([FB_WITH_EXTERNAL_PATH], +[ +cdir=`pwd` +AC_MSG_CHECKING([Checking EXTERNAL_PATH set to]) +AC_ARG_WITH([externalpath], + [ --with-externalpath=DIR User specified path to external facebook components.], + [ + if test "x${EXTERNAL_PATH}" != "x"; then + echo "" + echo "ERROR: You have already set EXTERNAL_PATH in your environment" + echo "Cannot override it using --with-externalpath. Unset EXTERNAL_PATH to use this option" + exit 1 + fi + EXTERNAL_PATH=$withval + ], + [ + if test "x${EXTERNAL_PATH}" = "x"; then + EXTERNAL_PATH=$1 + fi + ] +) +if test "x${EXTERNAL_PATH}" = "x"; then + export EXTERNAL_PATH="$cdir/external" + GLOBAL_HEADER_MK="include ${EXTERNAL_PATH}/global_header.mk" + GLOBAL_FOOTER_MK="include ${EXTERNAL_PATH}/global_footer.mk" +else + export EXTERNAL_PATH + GLOBAL_HEADER_MK="include ${EXTERNAL_PATH}/global_header.mk" + GLOBAL_FOOTER_MK="include ${EXTERNAL_PATH}/global_footer.mk" +fi +AC_MSG_RESULT($EXTERNAL_PATH) +if test ! -d ${EXTERNAL_PATH}; then + echo "" + echo "ERROR: EXTERNAL_PATH set to an nonexistent directory ${EXTERNAL_PATH}" + exit 1 +fi +AC_SUBST(EXTERNAL_PATH) +AC_SUBST(GLOBAL_HEADER_MK) +AC_SUBST(GLOBAL_FOOTER_MK) +]) + +# Set option to enable shared mode. Set DEBUG and OPT for use in Makefile.am. +AC_DEFUN([FB_ENABLE_DEFAULT_OPT_BUILD], +[ +AC_MSG_CHECKING([whether to enable optimized build]) +AC_ARG_ENABLE([opt], + [ --disable-opt Set up debug mode.], + [ + ENABLED_OPT=$enableval + ], + [ + ENABLED_OPT="yes" + ] +) +if test "$ENABLED_OPT" = "yes" +then + CFLAGS="-Wall -O3" + CXXFLAGS="-Wall -O3" +else + CFLAGS="-Wall -g" + CXXFLAGS="-Wall -g" +fi +AC_MSG_RESULT($ENABLED_OPT) +AM_CONDITIONAL([OPT], [test "$ENABLED_OPT" = yes]) +AM_CONDITIONAL([DEBUG], [test "$ENABLED_OPT" = no]) +]) + +# Set option to enable debug mode. Set DEBUG and OPT for use in Makefile.am. +AC_DEFUN([FB_ENABLE_DEFAULT_DEBUG_BUILD], +[ +AC_MSG_CHECKING([whether to enable debug build]) +AC_ARG_ENABLE([debug], + [ --disable-debug Set up opt mode.], + [ + ENABLED_DEBUG=$enableval + ], + [ + ENABLED_DEBUG="yes" + ] +) +if test "$ENABLED_DEBUG" = "yes" +then + CFLAGS="-Wall -g" + CXXFLAGS="-Wall -g" +else + CFLAGS="-Wall -O3" + CXXFLAGS="-Wall -O3" +fi +AC_MSG_RESULT($ENABLED_DEBUG) +AM_CONDITIONAL([DEBUG], [test "$ENABLED_DEBUG" = yes]) +AM_CONDITIONAL([OPT], [test "$ENABLED_DEBUG" = no]) +]) + +# Set option to enable static libs. +AC_DEFUN([FB_ENABLE_DEFAULT_STATIC], +[ +SHARED="" +STATIC="" +AC_MSG_CHECKING([whether to enable static mode]) +AC_ARG_ENABLE([static], + [ --disable-static Set up shared mode.], + [ + ENABLED_STATIC=$enableval + ], + [ + ENABLED_STATIC="yes" + ] +) +if test "$ENABLED_STATIC" = "yes" +then + LTYPE=".a" +else + LTYPE=".so" + SHARED_CXXFLAGS="-fPIC" + SHARED_CFLAGS="-fPIC" + SHARED_LDFLAGS="-shared -fPIC" + AC_SUBST(SHARED_CXXFLAGS) + AC_SUBST(SHARED_CFLAGS) + AC_SUBST(SHARED_LDFLAGS) +fi +AC_MSG_RESULT($ENABLED_STATIC) +AC_SUBST(LTYPE) +AM_CONDITIONAL([STATIC], [test "$ENABLED_STATIC" = yes]) +AM_CONDITIONAL([SHARED], [test "$ENABLED_STATIC" = no]) +]) + +# Set option to enable shared libs. +AC_DEFUN([FB_ENABLE_DEFAULT_SHARED], +[ +SHARED="" +STATIC="" +AC_MSG_CHECKING([whether to enable shared mode]) +AC_ARG_ENABLE([shared], + [ --disable-shared Set up static mode.], + [ + ENABLED_SHARED=$enableval + ], + [ + ENABLED_SHARED="yes" + ] +) +if test "$ENABLED_SHARED" = "yes" +then + LTYPE=".so" + SHARED_CXXFLAGS="-fPIC" + SHARED_CFLAGS="-fPIC" + SHARED_LDFLAGS="-shared -fPIC" + AC_SUBST(SHARED_CXXFLAGS) + AC_SUBST(SHARED_CFLAGS) + AC_SUBST(SHARED_LDFLAGS) +else + LTYPE=".a" +fi +AC_MSG_RESULT($ENABLED_SHARED) +AC_SUBST(LTYPE) +AM_CONDITIONAL([SHARED], [test "$ENABLED_SHARED" = yes]) +AM_CONDITIONAL([STATIC], [test "$ENABLED_SHARED" = no]) +]) + +# Generates define flags and conditionals as specified by user. +# This gets enabled *only* if user selects --enable- otion. +AC_DEFUN([FB_ENABLE_FEATURE], +[ +ENABLE="" +flag="$1" +value="$3" +AC_MSG_CHECKING([whether to enable $1]) +AC_ARG_ENABLE([$2], + [ --enable-$2 Enable $2.], + [ + ENABLE=$enableval + ], + [ + ENABLE="no" + ] +) +AM_CONDITIONAL([$1], [test "$ENABLE" = yes]) +if test "$ENABLE" = "yes" +then + if test "x${value}" = "x" + then + AC_DEFINE([$1]) + else + AC_DEFINE_UNQUOTED([$1], [$value]) + fi +fi +AC_MSG_RESULT($ENABLE) +]) + + +# can also use eval $2=$withval;AC_SUBST($2) +AC_DEFUN([FB_WITH_PATH], +[ +USRFLAG="" +USRFLAG=$1 +AC_MSG_CHECKING([Checking $1 set to]) +AC_ARG_WITH([$2], + [ --with-$2=DIR User specified path.], + [ + LOC=$withval + eval $USRFLAG=$withval + ], + [ + LOC=$3 + eval $USRFLAG=$3 + ] +) +AC_SUBST([$1]) +AC_MSG_RESULT($LOC) +]) + +AC_DEFUN([FB_SET_FLAG_VALUE], +[ +SETFLAG="" +AC_MSG_CHECKING([Checking $1 set to]) +SETFLAG=$1 +eval $SETFLAG=\"$2\" +AC_SUBST([$SETFLAG]) +AC_MSG_RESULT($2) +]) + +# NOTES +# if using if else bourne stmt you must have more than a macro in it. +# EX1 is not correct. EX2 is correct +# EX1: if test "$XX" = "yes"; then +# AC_SUBST(xx) +# fi +# EX2: if test "$XX" = "yes"; then +# xx="foo" +# AC_SUBST(xx) +# fi diff --git a/contrib/fb303/aclocal/ax_boost_base.m4 b/contrib/fb303/aclocal/ax_boost_base.m4 new file mode 100644 index 00000000..e56bb738 --- /dev/null +++ b/contrib/fb303/aclocal/ax_boost_base.m4 @@ -0,0 +1,198 @@ +##### http://autoconf-archive.cryp.to/ax_boost_base.html +# +# SYNOPSIS +# +# AX_BOOST_BASE([MINIMUM-VERSION]) +# +# DESCRIPTION +# +# Test for the Boost C++ libraries of a particular version (or newer) +# +# If no path to the installed boost library is given the macro +# searchs under /usr, /usr/local, /opt and /opt/local and evaluates +# the $BOOST_ROOT environment variable. Further documentation is +# available at . +# +# This macro calls: +# +# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS) +# +# And sets: +# +# HAVE_BOOST +# +# LAST MODIFICATION +# +# 2007-07-28 +# +# COPYLEFT +# +# Copyright (c) 2007 Thomas Porschberg +# +# Copying and distribution of this file, with or without +# modification, are permitted in any medium without royalty provided +# the copyright notice and this notice are preserved. + +AC_DEFUN([AX_BOOST_BASE], +[ +AC_ARG_WITH([boost], + AS_HELP_STRING([--with-boost@<:@=DIR@:>@], [use boost (default is yes) - it is possible to specify the root directory for boost (optional)]), + [ + if test "$withval" = "no"; then + want_boost="no" + elif test "$withval" = "yes"; then + want_boost="yes" + ac_boost_path="" + else + want_boost="yes" + ac_boost_path="$withval" + fi + ], + [want_boost="yes"]) + +if test "x$want_boost" = "xyes"; then + boost_lib_version_req=ifelse([$1], ,1.20.0,$1) + boost_lib_version_req_shorten=`expr $boost_lib_version_req : '\([[0-9]]*\.[[0-9]]*\)'` + boost_lib_version_req_major=`expr $boost_lib_version_req : '\([[0-9]]*\)'` + boost_lib_version_req_minor=`expr $boost_lib_version_req : '[[0-9]]*\.\([[0-9]]*\)'` + boost_lib_version_req_sub_minor=`expr $boost_lib_version_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'` + if test "x$boost_lib_version_req_sub_minor" = "x" ; then + boost_lib_version_req_sub_minor="0" + fi + WANT_BOOST_VERSION=`expr $boost_lib_version_req_major \* 100000 \+ $boost_lib_version_req_minor \* 100 \+ $boost_lib_version_req_sub_minor` + AC_MSG_CHECKING(for boostlib >= $boost_lib_version_req) + succeeded=no + + dnl first we check the system location for boost libraries + dnl this location ist chosen if boost libraries are installed with the --layout=system option + dnl or if you install boost with RPM + if test "$ac_boost_path" != ""; then + BOOST_LDFLAGS="-L$ac_boost_path/lib" + BOOST_CPPFLAGS="-I$ac_boost_path/include" + else + for ac_boost_path_tmp in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path_tmp/include/boost" && test -r "$ac_boost_path_tmp/include/boost"; then + BOOST_LDFLAGS="-L$ac_boost_path_tmp/lib" + BOOST_CPPFLAGS="-I$ac_boost_path_tmp/include" + break; + fi + done + fi + + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_LANG_PUSH(C++) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + succeeded=yes + found_system=yes + ],[ + ]) + AC_LANG_POP([C++]) + + + + dnl if we found no boost with system layout we search for boost libraries + dnl built and installed without the --layout=system option or for a staged(not installed) version + if test "x$succeeded" != "xyes"; then + _version=0 + if test "$ac_boost_path" != ""; then + BOOST_LDFLAGS="-L$ac_boost_path/lib" + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + fi + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$ac_boost_path/include/boost-$VERSION_UNDERSCORE" + done + fi + else + for ac_boost_path in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + best_path=$ac_boost_path + fi + done + fi + done + + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE" + BOOST_LDFLAGS="-L$best_path/lib" + + if test "x$BOOST_ROOT" != "x"; then + if test -d "$BOOST_ROOT" && test -r "$BOOST_ROOT" && test -d "$BOOST_ROOT/stage/lib" && test -r "$BOOST_ROOT/stage/lib"; then + version_dir=`expr //$BOOST_ROOT : '.*/\(.*\)'` + stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'` + stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'` + V_CHECK=`expr $stage_version_shorten \>\= $_version` + if test "$V_CHECK" = "1" ; then + AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT) + BOOST_CPPFLAGS="-I$BOOST_ROOT" + BOOST_LDFLAGS="-L$BOOST_ROOT/stage/lib" + fi + fi + fi + fi + + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_LANG_PUSH(C++) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + succeeded=yes + found_system=yes + ],[ + ]) + AC_LANG_POP([C++]) + fi + + if test "$succeeded" != "yes" ; then + if test "$_version" = "0" ; then + AC_MSG_ERROR([[We could not detect the boost libraries (version $boost_lib_version_req_shorten or higher). If you have a staged boost library (still not installed) please specify \$BOOST_ROOT in your environment and do not give a PATH to --with-boost option. If you are sure you have boost installed, then check your version number looking in . See http://randspringer.de/boost for more documentation.]]) + else + AC_MSG_NOTICE([Your boost libraries seems to old (version $_version).]) + fi + else + AC_SUBST(BOOST_CPPFLAGS) + AC_SUBST(BOOST_LDFLAGS) + AC_DEFINE(HAVE_BOOST,,[define if the Boost library is available]) + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" +fi + +]) diff --git a/contrib/fb303/bootstrap.sh b/contrib/fb303/bootstrap.sh new file mode 100755 index 00000000..3cbeddb3 --- /dev/null +++ b/contrib/fb303/bootstrap.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# To be safe include -I flag +aclocal -I ./aclocal +automake -a +autoconf +./configure --config-cache $* diff --git a/contrib/fb303/configure.ac b/contrib/fb303/configure.ac new file mode 100644 index 00000000..67cc1108 --- /dev/null +++ b/contrib/fb303/configure.ac @@ -0,0 +1,115 @@ +# Autoconf input file +# $Id$ + +# AC - autoconf +# FB - facebook + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +######################################################################### +# DO NOT TOUCH EXCEPT TO CHANGE REV# IN AC_INIT + +AC_PREREQ(2.52) +AC_INIT([libfb303],[20080209]) +#AC_CONFIG_AUX_DIR([/usr/share/automake-1.9]) +# To install locally +FB_INITIALIZE([localinstall]) +AC_PREFIX_DEFAULT([/usr/local]) + +############################################################################ +# User Configurable. Change With CAUTION! +# User can include custom makefile rules. Uncomment and update only in PRODUCT_MK. +# Include where appropriate in any Makefile.am as @PRODUCT_MK@ + +#PRODUCT_MK="include ${EXTERNAL_PATH}/shared/build/.mk" + +# Default path to external Facebook components and shared build toools I.e fb303 etc. +# To point to other locations set environment variable EXTERNAL_PATH. +# To change the current default you must change bootstrap.sh. +FB_WITH_EXTERNAL_PATH([`pwd`]) + +AC_ARG_VAR([PY_PREFIX], [Prefix for installing Python modules. + (Normal --prefix is ignored for Python because + Python has different conventions.) + Default = "/usr"]) +AS_IF([test "x$PY_PREFIX" = x], [PY_PREFIX="/usr"]) + +########################################################################## +# User Configurable + +# Pre-defined macro to set opt build mode. Run with --disable-shared option to turn off optimization. +FB_ENABLE_DEFAULT_OPT_BUILD + +# Predefined macro to set static library mode. Run with --disable-static option to turn off static lib mode. +FB_ENABLE_DEFAULT_STATIC + +# Personalized feature generator. Creates defines/conditionals and --enable --disable command line options. +# FB_ENABLE_FEATURE([FEATURE], [feature]) OR FB_ENABLE_FEATURE([FEATURE], [feature], [\"\"]) + +# Example: Macro supplies -DFACEBOOK at compile time and "if FACEBOOK endif" capabilities. + +# Personalized path generator Sets default paths. Provides --with-xx=DIR options. +# FB_WITH_PATH([_home], [path], [] + +# Example: sets $(thrift_home) variable with default path set to /usr/local. +FB_WITH_PATH([thrift_home], [thriftpath], [/usr/local]) + +# Require boost 1.33.1 or later +AX_BOOST_BASE([1.33.1]) + +# Generates Makefile from Makefile.am. Modify when new subdirs are added. +# Change Makefile.am also to add subdirectly. +AC_CONFIG_FILES(Makefile cpp/Makefile py/Makefile) + +############################################################################ +# DO NOT TOUCH. + +AC_SUBST(PRODUCT_MK) +AC_OUTPUT + +############################################################################# +######### FINISH ############################################################ + +echo "EXTERNAL_PATH $EXTERNAL_PATH" + + +# +# NOTES FOR USER +# Short cut to create conditional flags. +#enable_facebook="yes" +#AM_CONDITIONAL([FACEBOOK], [test "$enable_facebook" = yes]) +#enable_hdfs="yes" +#AM_CONDITIONAL([HDFS], [test "$enable_hdfs" = yes]) + +# Enable options with --enable and --disable configurable. +#AC_MSG_CHECKING([whether to enable FACEBOOK]) +#FACEBOOK="" +#AC_ARG_ENABLE([facebook], +# [ --enable-facebook Enable facebook.], +# [ +# ENABLE_FACEBOOK=$enableval +# ], +# [ +# ENABLE_FACEBOOK="no" +# ] +#) +#AM_CONDITIONAL([FACEBOOK], [test "$ENABLE_FACEBOOK" = yes]) +#AC_MSG_RESULT($ENABLE_FACEBOOK) + diff --git a/contrib/fb303/cpp/FacebookBase.cpp b/contrib/fb303/cpp/FacebookBase.cpp new file mode 100644 index 00000000..80033406 --- /dev/null +++ b/contrib/fb303/cpp/FacebookBase.cpp @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "FacebookBase.h" + +using namespace facebook::fb303; +using apache::thrift::concurrency::Guard; + +FacebookBase::FacebookBase(std::string name) : + name_(name) { + aliveSince_ = (int64_t) time(NULL); +} + +inline void FacebookBase::getName(std::string& _return) { + _return = name_; +} + +void FacebookBase::setOption(const std::string& key, const std::string& value) { + Guard g(optionsLock_); + options_[key] = value; +} + +void FacebookBase::getOption(std::string& _return, const std::string& key) { + Guard g(optionsLock_); + _return = options_[key]; +} + +void FacebookBase::getOptions(std::map & _return) { + Guard g(optionsLock_); + _return = options_; +} + +int64_t FacebookBase::incrementCounter(const std::string& key, int64_t amount) { + counters_.acquireRead(); + + // if we didn't find the key, we need to write lock the whole map to create it + ReadWriteCounterMap::iterator it = counters_.find(key); + if (it == counters_.end()) { + counters_.release(); + counters_.acquireWrite(); + + // we need to check again to make sure someone didn't create this key + // already while we released the lock + it = counters_.find(key); + if(it == counters_.end()){ + counters_[key].value = amount; + counters_.release(); + return amount; + } + } + + it->second.acquireWrite(); + int64_t count = it->second.value + amount; + it->second.value = count; + it->second.release(); + counters_.release(); + return count; +} + +int64_t FacebookBase::setCounter(const std::string& key, int64_t value) { + counters_.acquireRead(); + + // if we didn't find the key, we need to write lock the whole map to create it + ReadWriteCounterMap::iterator it = counters_.find(key); + if (it == counters_.end()) { + counters_.release(); + counters_.acquireWrite(); + counters_[key].value = value; + counters_.release(); + return value; + } + + it->second.acquireWrite(); + it->second.value = value; + it->second.release(); + counters_.release(); + return value; +} + +void FacebookBase::getCounters(std::map& _return) { + // we need to lock the whole thing and actually build the map since we don't + // want our read/write structure to go over the wire + counters_.acquireRead(); + for(ReadWriteCounterMap::iterator it = counters_.begin(); + it != counters_.end(); it++) + { + _return[it->first] = it->second.value; + } + counters_.release(); +} + +int64_t FacebookBase::getCounter(const std::string& key) { + int64_t rv = 0; + counters_.acquireRead(); + ReadWriteCounterMap::iterator it = counters_.find(key); + if (it != counters_.end()) { + it->second.acquireRead(); + rv = it->second.value; + it->second.release(); + } + counters_.release(); + return rv; +} + +inline int64_t FacebookBase::aliveSince() { + return aliveSince_; +} + diff --git a/contrib/fb303/cpp/FacebookBase.h b/contrib/fb303/cpp/FacebookBase.h new file mode 100644 index 00000000..fd169e62 --- /dev/null +++ b/contrib/fb303/cpp/FacebookBase.h @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _FACEBOOK_TB303_FACEBOOKBASE_H_ +#define _FACEBOOK_TB303_FACEBOOKBASE_H_ 1 + +#include "FacebookService.h" + +#include "server/TServer.h" +#include "concurrency/Mutex.h" + +#include +#include +#include + +namespace facebook { namespace fb303 { + +using apache::thrift::concurrency::Mutex; +using apache::thrift::concurrency::ReadWriteMutex; +using apache::thrift::server::TServer; + +struct ReadWriteInt : ReadWriteMutex {int64_t value;}; +struct ReadWriteCounterMap : ReadWriteMutex, + std::map {}; + +/** + * Base Facebook service implementation in C++. + * + */ +class FacebookBase : virtual public FacebookServiceIf { + protected: + FacebookBase(std::string name); + virtual ~FacebookBase() {} + + public: + void getName(std::string& _return); + virtual void getVersion(std::string& _return) { _return = ""; } + + virtual fb_status getStatus() = 0; + virtual void getStatusDetails(std::string& _return) { _return = ""; } + + void setOption(const std::string& key, const std::string& value); + void getOption(std::string& _return, const std::string& key); + void getOptions(std::map & _return); + + int64_t aliveSince(); + + virtual void reinitialize() {} + + virtual void shutdown() { + if (server_.get() != NULL) { + server_->stop(); + } + } + + int64_t incrementCounter(const std::string& key, int64_t amount = 1); + int64_t setCounter(const std::string& key, int64_t value); + + void getCounters(std::map& _return); + int64_t getCounter(const std::string& key); + + /** + * Set server handle for shutdown method + */ + void setServer(boost::shared_ptr server) { + server_ = server; + } + + void getCpuProfile(std::string& _return, int32_t durSecs) { _return = ""; } + + private: + + std::string name_; + int64_t aliveSince_; + + std::map options_; + Mutex optionsLock_; + + ReadWriteCounterMap counters_; + + boost::shared_ptr server_; + +}; + +}} // facebook::tb303 + +#endif // _FACEBOOK_TB303_FACEBOOKBASE_H_ diff --git a/contrib/fb303/cpp/Makefile.am b/contrib/fb303/cpp/Makefile.am new file mode 100644 index 00000000..e62608ce --- /dev/null +++ b/contrib/fb303/cpp/Makefile.am @@ -0,0 +1,84 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +@GLOBAL_HEADER_MK@ + +@PRODUCT_MK@ + + +# User specified path variables set in configure.ac. +# thrift_home +# +THRIFT = $(thrift_home)/bin/thrift + +# User defined conditionals and conditonal statements set up in configure.ac. +if DEBUG + DEBUG_CPPFLAGS = -DDEBUG_TIMING +endif + +# Set common flags recognized by automake. +# DO NOT USE CPPFLAGS, CXXFLAGS, CFLAGS, LDFLAGS here! Set in configure.ac and|or override on command line. +# USE flags AM_CXXFLAGS, AM_CFLAGS, AM_CPPFLAGS, AM_LDFLAGS, LDADD in this section. + +AM_CPPFLAGS = -I.. +AM_CPPFLAGS += -Igen-cpp +AM_CPPFLAGS += -I$(thrift_home)/include/thrift +AM_CPPFLAGS += $(BOOST_CPPFLAGS) +AM_CPPFLAGS += $(FB_CPPFLAGS) $(DEBUG_CPPFLAGS) + +# GENERATE BUILD RULES +# Set Program/library specific flags recognized by automake. +# Use _ to set prog / lib specific flag s +# foo_CXXFLAGS foo_CPPFLAGS foo_LDFLAGS foo_LDADD + +fb303_lib = gen-cpp/FacebookService.cpp gen-cpp/fb303_constants.cpp gen-cpp/fb303_types.cpp FacebookBase.cpp ServiceTracker.cpp + +# Static -- multiple libraries can be defined +if STATIC +lib_LIBRARIES = libfb303.a +libfb303_a_SOURCES = $(fb303_lib) +INTERNAL_LIBS = libfb303.a +endif + +# Shared -- multiple libraries can be defined +if SHARED +shareddir = lib +shared_PROGRAMS = libfb303.so +libfb303_so_SOURCES = $(fb303_lib) +libfb303_so_CXXFLAGS = $(SHARED_CXXFLAGS) +libfb303_so_LDFLAGS = $(SHARED_LDFLAGS) +INTERNAL_LIBS = libfb303.so +endif + +# Set up Thrift specific activity here. +# We assume that a +types.cpp will always be built from .thrift. +$(eval $(call thrift_template,.,../if/fb303.thrift,-I $(thrift_home)/share --gen cpp )) + +include_fb303dir = $(includedir)/thrift/fb303 +include_fb303_HEADERS = FacebookBase.h ServiceTracker.h gen-cpp/FacebookService.h gen-cpp/fb303_constants.h gen-cpp/fb303_types.h + +include_fb303ifdir = $(prefix)/share/fb303/if +include_fb303if_HEADERS = ../if/fb303.thrift + +BUILT_SOURCES = thriftstyle + +# Add to pre-existing target clean +clean-local: clean-common + +@GLOBAL_FOOTER_MK@ diff --git a/contrib/fb303/cpp/ServiceTracker.cpp b/contrib/fb303/cpp/ServiceTracker.cpp new file mode 100644 index 00000000..c20a0683 --- /dev/null +++ b/contrib/fb303/cpp/ServiceTracker.cpp @@ -0,0 +1,481 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include "FacebookBase.h" +#include "ServiceTracker.h" +#include "concurrency/ThreadManager.h" + +using namespace std; +using namespace facebook::fb303; +using namespace apache::thrift::concurrency; + + +uint64_t ServiceTracker::CHECKPOINT_MINIMUM_INTERVAL_SECONDS = 60; +int ServiceTracker::LOG_LEVEL = 5; + + +ServiceTracker::ServiceTracker(facebook::fb303::FacebookBase *handler, + void (*logMethod)(int, const string &), + bool featureCheckpoint, + bool featureStatusCheck, + bool featureThreadCheck, + Stopwatch::Unit stopwatchUnit) + : handler_(handler), logMethod_(logMethod), + featureCheckpoint_(featureCheckpoint), + featureStatusCheck_(featureStatusCheck), + featureThreadCheck_(featureThreadCheck), + stopwatchUnit_(stopwatchUnit), + checkpointServices_(0) +{ + if (featureCheckpoint_) { + time_t now = time(NULL); + checkpointTime_ = now; + } else { + checkpointTime_ = 0; + } +} + +/** + * Registers the beginning of a "service method": basically, any of + * the implementations of Thrift remote procedure calls that a + * FacebookBase handler is handling. Controls concurrent + * services and reports statistics (via log and via fb303 counters). + * Throws an exception if the server is not ready to handle service + * methods yet. + * + * note: The relationship between startService() and finishService() + * is currently defined so that a call to finishService() should only + * be matched to this call to startService() if this method returns + * without exception. It wouldn't be a problem to implement things + * the other way, so that *every* start needed a finish, but this + * convention was chosen to match the way an object's constructor and + * destructor work together, i.e. to work well with ServiceMethod + * objects. + * + * @param const ServiceMethod &serviceMethod A reference to the ServiceMethod + * object instantiated at the start + * of the service method. + */ +void +ServiceTracker::startService(const ServiceMethod &serviceMethod) +{ + // note: serviceMethod.timer_ automatically starts at construction. + + // log service start + logMethod_(5, serviceMethod.signature_); + + // check handler ready + if (featureStatusCheck_ && !serviceMethod.featureLogOnly_) { + // note: Throwing exceptions before counting statistics. See note + // in method header. + // note: A STOPPING server is not accepting new connections, but it + // is still handling any already-connected threads -- so from the + // service method's point of view, a status of STOPPING is a green + // light. + facebook::fb303::fb_status status = handler_->getStatus(); + if (status != facebook::fb303::ALIVE + && status != facebook::fb303::STOPPING) { + if (status == facebook::fb303::STARTING) { + throw ServiceException("Server starting up; please try again later"); + } else { + throw ServiceException("Server not alive; please try again later"); + } + } + } + + // check server threads + if (featureThreadCheck_ && !serviceMethod.featureLogOnly_) { + // note: Might want to put these messages in reportCheckpoint() if + // log is getting spammed. + if (threadManager_ != NULL) { + size_t idle_count = threadManager_->idleWorkerCount(); + if (idle_count == 0) { + stringstream message; + message << "service " << serviceMethod.signature_ + << ": all threads (" << threadManager_->workerCount() + << ") in use"; + logMethod_(3, message.str()); + } + } + } +} + +/** + * Logs a significant step in the middle of a "service method"; see + * startService. + * + * @param const ServiceMethod &serviceMethod A reference to the ServiceMethod + * object instantiated at the start + * of the service method. + * @return int64_t Elapsed units (see stopwatchUnit_) since ServiceMethod + * instantiation. + */ +int64_t +ServiceTracker::stepService(const ServiceMethod &serviceMethod, + const string &stepName) +{ + stringstream message; + string elapsed_label; + int64_t elapsed = serviceMethod.timer_.elapsedUnits(stopwatchUnit_, + &elapsed_label); + message << serviceMethod.signature_ + << ' ' << stepName + << " [" << elapsed_label << ']'; + logMethod_(5, message.str()); + return elapsed; +} + +/** + * Registers the end of a "service method"; see startService(). + * + * @param const ServiceMethod &serviceMethod A reference to the ServiceMethod + * object instantiated at the start + * of the service method. + */ +void +ServiceTracker::finishService(const ServiceMethod &serviceMethod) +{ + // log end of service + stringstream message; + string duration_label; + int64_t duration = serviceMethod.timer_.elapsedUnits(stopwatchUnit_, + &duration_label); + message << serviceMethod.signature_ + << " finish [" << duration_label << ']'; + logMethod_(5, message.str()); + + // count, record, and maybe report service statistics + if (!serviceMethod.featureLogOnly_) { + + if (!featureCheckpoint_) { + + // lifetime counters + // (note: No need to lock statisticsMutex_ if not doing checkpoint; + // FacebookService::incrementCounter() is already thread-safe.) + handler_->incrementCounter("lifetime_services"); + + } else { + + statisticsMutex_.lock(); + // note: No exceptions expected from this code block. Wrap in a try + // just to be safe. + try { + + // lifetime counters + // note: Good to synchronize this with the increment of + // checkpoint services, even though incrementCounter() is + // already thread-safe, for the sake of checkpoint reporting + // consistency (i.e. since the last checkpoint, + // lifetime_services has incremented by checkpointServices_). + handler_->incrementCounter("lifetime_services"); + + // checkpoint counters + checkpointServices_++; + checkpointDuration_ += duration; + + // per-service timing + // note kjv: According to my tests it is very slightly faster to + // call insert() once (and detect not-found) than calling find() + // and then maybe insert (if not-found). However, the difference + // is tiny for small maps like this one, and the code for the + // faster solution is slightly less readable. Also, I wonder if + // the instantiation of the (often unused) pair to insert makes + // the first algorithm slower after all. + map >::iterator iter; + iter = checkpointServiceDuration_.find(serviceMethod.name_); + if (iter != checkpointServiceDuration_.end()) { + iter->second.first++; + iter->second.second += duration; + } else { + checkpointServiceDuration_.insert(make_pair(serviceMethod.name_, + make_pair(1, duration))); + } + + // maybe report checkpoint + // note: ...if it's been long enough since the last report. + time_t now = time(NULL); + uint64_t check_interval = now - checkpointTime_; + if (check_interval >= CHECKPOINT_MINIMUM_INTERVAL_SECONDS) { + reportCheckpoint(); + } + + } catch (...) { + statisticsMutex_.unlock(); + throw; + } + statisticsMutex_.unlock(); + + } + } +} + +/** + * Logs some statistics gathered since the last call to this method. + * + * note: Thread race conditions on this method could cause + * misreporting and/or undefined behavior; the caller must protect + * uses of the object variables (and calls to this method) with a + * mutex. + * + */ +void +ServiceTracker::reportCheckpoint() +{ + time_t now = time(NULL); + + uint64_t check_count = checkpointServices_; + uint64_t check_interval = now - checkpointTime_; + uint64_t check_duration = checkpointDuration_; + + // export counters for timing of service methods (by service name) + handler_->setCounter("checkpoint_time", check_interval); + map >::iterator iter; + uint64_t count; + for (iter = checkpointServiceDuration_.begin(); + iter != checkpointServiceDuration_.end(); + iter++) { + count = iter->second.first; + handler_->setCounter(string("checkpoint_count_") + iter->first, count); + if (count == 0) { + handler_->setCounter(string("checkpoint_speed_") + iter->first, + 0); + } else { + handler_->setCounter(string("checkpoint_speed_") + iter->first, + iter->second.second / count); + } + } + + // reset checkpoint variables + // note: Clearing the map while other threads are using it might + // cause undefined behavior. + checkpointServiceDuration_.clear(); + checkpointTime_ = now; + checkpointServices_ = 0; + checkpointDuration_ = 0; + + // get lifetime variables + uint64_t life_count = handler_->getCounter("lifetime_services"); + uint64_t life_interval = now - handler_->aliveSince(); + + // log checkpoint + stringstream message; + message << "checkpoint_time:" << check_interval + << " checkpoint_services:" << check_count + << " checkpoint_speed_sum:" << check_duration + << " lifetime_time:" << life_interval + << " lifetime_services:" << life_count; + if (featureThreadCheck_ && threadManager_ != NULL) { + size_t worker_count = threadManager_->workerCount(); + size_t idle_count = threadManager_->idleWorkerCount(); + message << " total_workers:" << worker_count + << " active_workers:" << (worker_count - idle_count); + } + logMethod_(4, message.str()); +} + +/** + * Remembers the thread manager used in the server, for monitoring thread + * activity. + * + * @param shared_ptr threadManager The server's thread manager. + */ +void +ServiceTracker::setThreadManager(boost::shared_ptr + threadManager) +{ + threadManager_ = threadManager; +} + +/** + * Logs messages to stdout; the passed message will be logged if the + * passed level is less than or equal to LOG_LEVEL. + * + * This is the default logging method used by the ServiceTracker. An + * alternate logging method (that accepts the same parameters) may be + * specified to the constructor. + * + * @param int level A level associated with the message: higher levels + * are used to indicate higher levels of detail. + * @param string message The message to log. + */ +void +ServiceTracker::defaultLogMethod(int level, const string &message) +{ + if (level <= LOG_LEVEL) { + string level_string; + time_t now = time(NULL); + char now_pretty[26]; + ctime_r(&now, now_pretty); + now_pretty[24] = '\0'; + switch (level) { + case 1: + level_string = "CRITICAL"; + break; + case 2: + level_string = "ERROR"; + break; + case 3: + level_string = "WARNING"; + break; + case 5: + level_string = "DEBUG"; + break; + case 4: + default: + level_string = "INFO"; + break; + } + cout << '[' << level_string << "] [" << now_pretty << "] " + << message << endl; + } +} + + +/** + * Creates a Stopwatch, which can report the time elapsed since its + * creation. + * + */ +Stopwatch::Stopwatch() +{ + gettimeofday(&startTime_, NULL); +} + +void +Stopwatch::reset() +{ + gettimeofday(&startTime_, NULL); +} + +uint64_t +Stopwatch::elapsedUnits(Stopwatch::Unit unit, string *label) const +{ + timeval now_time; + gettimeofday(&now_time, NULL); + time_t duration_secs = now_time.tv_sec - startTime_.tv_sec; + + uint64_t duration_units; + switch (unit) { + case UNIT_SECONDS: + duration_units = duration_secs + + (now_time.tv_usec - startTime_.tv_usec + 500000) / 1000000; + if (NULL != label) { + stringstream ss_label; + ss_label << duration_units << " secs"; + label->assign(ss_label.str()); + } + break; + case UNIT_MICROSECONDS: + duration_units = duration_secs * 1000000 + + now_time.tv_usec - startTime_.tv_usec; + if (NULL != label) { + stringstream ss_label; + ss_label << duration_units << " us"; + label->assign(ss_label.str()); + } + break; + case UNIT_MILLISECONDS: + default: + duration_units = duration_secs * 1000 + + (now_time.tv_usec - startTime_.tv_usec + 500) / 1000; + if (NULL != label) { + stringstream ss_label; + ss_label << duration_units << " ms"; + label->assign(ss_label.str()); + } + break; + } + return duration_units; +} + +/** + * Creates a ServiceMethod, used for tracking a single service method + * invocation (via the ServiceTracker). The passed name of the + * ServiceMethod is used to group statistics (e.g. counts and durations) + * for similar invocations; the passed signature is used to uniquely + * identify the particular invocation in the log. + * + * note: A version of this constructor is provided that automatically + * forms a signature the name and a passed numeric id. Silly, sure, + * but commonly used, since it often saves the caller a line or two of + * code. + * + * @param ServiceTracker *tracker The service tracker that will track this + * ServiceMethod. + * @param const string &name The service method name (usually independent + * of service method parameters). + * @param const string &signature A signature uniquely identifying the method + * invocation (usually name plus parameters). + */ +ServiceMethod::ServiceMethod(ServiceTracker *tracker, + const string &name, + const string &signature, + bool featureLogOnly) + : tracker_(tracker), name_(name), signature_(signature), + featureLogOnly_(featureLogOnly) +{ + // note: timer_ automatically starts at construction. + + // invoke tracker to start service + // note: Might throw. If it throws, then this object's destructor + // won't be called, which is according to plan: finishService() is + // only supposed to be matched to startService() if startService() + // returns without error. + tracker_->startService(*this); +} + +ServiceMethod::ServiceMethod(ServiceTracker *tracker, + const string &name, + uint64_t id, + bool featureLogOnly) + : tracker_(tracker), name_(name), featureLogOnly_(featureLogOnly) +{ + // note: timer_ automatically starts at construction. + stringstream ss_signature; + ss_signature << name << " (" << id << ')'; + signature_ = ss_signature.str(); + + // invoke tracker to start service + // note: Might throw. If it throws, then this object's destructor + // won't be called, which is according to plan: finishService() is + // only supposed to be matched to startService() if startService() + // returns without error. + tracker_->startService(*this); +} + +ServiceMethod::~ServiceMethod() +{ + // invoke tracker to finish service + // note: Not expecting an exception from this code, but + // finishService() might conceivably throw an out-of-memory + // exception. + try { + tracker_->finishService(*this); + } catch (...) { + // don't throw + } +} + +uint64_t +ServiceMethod::step(const std::string &stepName) +{ + return tracker_->stepService(*this, stepName); +} diff --git a/contrib/fb303/cpp/ServiceTracker.h b/contrib/fb303/cpp/ServiceTracker.h new file mode 100644 index 00000000..93043863 --- /dev/null +++ b/contrib/fb303/cpp/ServiceTracker.h @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * ServiceTracker is a utility class for logging and timing service + * calls to a fb303 Thrift server. Currently, ServiceTracker offers + * the following features: + * + * . Logging of service method start, end (and duration), and + * optional steps in between. + * + * . Automatic check of server status via fb303::getStatus() + * with a ServiceException thrown if server not alive + * (at method start). + * + * . A periodic logged checkpoint reporting lifetime time, lifetime + * service count, and per-method statistics since the last checkpoint + * time (at method finish). + * + * . Export of fb303 counters for lifetime and checkpoint statistics + * (at method finish). + * + * . For TThreadPoolServers, a logged warning when all server threads + * are busy (at method start). (Must call setThreadManager() after + * ServiceTracker instantiation for this feature to be enabled.) + * + * Individual features may be enabled or disabled by arguments to the + * constructor. The constructor also accepts a pointer to a logging + * method -- if no pointer is passed, the tracker will log to stdout. + * + * ServiceTracker defines private methods for service start, finish, + * and step, which are designed to be accessed by instantiating a + * friend ServiceMethod object, as in the following example: + * + * #include + * class MyServiceHandler : virtual public MyServiceIf, + * public facebook::fb303::FacebookBase + * { + * public: + * MyServiceHandler::MyServiceHandler() : mServiceTracker(this) {} + * void MyServiceHandler::myServiceMethod(int userId) { + * // note: Instantiating a ServiceMethod object starts a timer + * // and tells the ServiceTracker to log the start. Might throw + * // a ServiceException. + * ServiceMethod serviceMethod(&mServiceTracker, + * "myServiceMethod", + * userId); + * ... + * // note: Calling the step method tells the ServiceTracker to + * // log the step, with a time elapsed since start. + * serviceMethod.step("post parsing, begin processing"); + * ... + * // note: When the ServiceMethod object goes out of scope, the + * // ServiceTracker will log the total elapsed time of the method. + * } + * ... + * private: + * ServiceTracker mServiceTracker; + * } + * + * The step() method call is optional; the startService() and + * finishService() methods are handled by the object's constructor and + * destructor. + * + * The ServiceTracker is (intended to be) thread-safe. + * + * Future: + * + * . Come up with something better for logging than passing a + * function pointer to the constructor. + * + * . Add methods for tracking errors from service methods, e.g. + * ServiceTracker::reportService(). + */ + +#ifndef SERVICETRACKER_H +#define SERVICETRACKER_H + + +#include +#include +#include +#include +#include +#include + +#include "concurrency/Mutex.h" + + +namespace apache { namespace thrift { namespace concurrency { + class ThreadManager; +}}} + + +namespace facebook { namespace fb303 { + + +class FacebookBase; +class ServiceMethod; + + +class Stopwatch +{ +public: + enum Unit { UNIT_SECONDS, UNIT_MILLISECONDS, UNIT_MICROSECONDS }; + Stopwatch(); + uint64_t elapsedUnits(Unit unit, std::string *label = NULL) const; + void reset(); +private: + timeval startTime_; +}; + + +class ServiceTracker +{ + friend class ServiceMethod; + +public: + + static uint64_t CHECKPOINT_MINIMUM_INTERVAL_SECONDS; + static int LOG_LEVEL; + + ServiceTracker(facebook::fb303::FacebookBase *handler, + void (*logMethod)(int, const std::string &) + = &ServiceTracker::defaultLogMethod, + bool featureCheckpoint = true, + bool featureStatusCheck = true, + bool featureThreadCheck = true, + Stopwatch::Unit stopwatchUnit + = Stopwatch::UNIT_MILLISECONDS); + + void setThreadManager(boost::shared_ptr threadManager); + +private: + + facebook::fb303::FacebookBase *handler_; + void (*logMethod_)(int, const std::string &); + boost::shared_ptr threadManager_; + + bool featureCheckpoint_; + bool featureStatusCheck_; + bool featureThreadCheck_; + Stopwatch::Unit stopwatchUnit_; + + apache::thrift::concurrency::Mutex statisticsMutex_; + time_t checkpointTime_; + uint64_t checkpointServices_; + uint64_t checkpointDuration_; + std::map > checkpointServiceDuration_; + + void startService(const ServiceMethod &serviceMethod); + int64_t stepService(const ServiceMethod &serviceMethod, + const std::string &stepName); + void finishService(const ServiceMethod &serviceMethod); + void reportCheckpoint(); + static void defaultLogMethod(int level, const std::string &message); +}; + + +class ServiceMethod +{ + friend class ServiceTracker; +public: + ServiceMethod(ServiceTracker *tracker, + const std::string &name, + const std::string &signature, + bool featureLogOnly = false); + ServiceMethod(ServiceTracker *tracker, + const std::string &name, + uint64_t id, + bool featureLogOnly = false); + ~ServiceMethod(); + uint64_t step(const std::string &stepName); +private: + ServiceTracker *tracker_; + std::string name_; + std::string signature_; + bool featureLogOnly_; + Stopwatch timer_; +}; + + +class ServiceException : public std::exception +{ +public: + explicit ServiceException(const std::string &message, int code = 0) + : message_(message), code_(code) {} + ~ServiceException() throw() {} + virtual const char *what() const throw() { return message_.c_str(); } + int code() const throw() { return code_; } +private: + std::string message_; + int code_; +}; + + +}} // facebook::fb303 + +#endif diff --git a/contrib/fb303/global_footer.mk b/contrib/fb303/global_footer.mk new file mode 100644 index 00000000..96f82ebd --- /dev/null +++ b/contrib/fb303/global_footer.mk @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +thriftstyle : $(XBUILT_SOURCES) + diff --git a/contrib/fb303/global_header.mk b/contrib/fb303/global_header.mk new file mode 100644 index 00000000..77c9455e --- /dev/null +++ b/contrib/fb303/global_header.mk @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +#define thrift_template +# $(1) : $(2) +# $$(THRIFT) $(3) $(4) $(5) $(6) $(7) $(8) $$< +#endef + +define thrift_template +XTARGET := $(shell perl -e '@val = split("\/","$(2)"); $$last = pop(@val);split("\\.",$$last);print "$(1)/"."gen-cpp/"."@_[0]"."_types.cpp\n"' ) + +ifneq ($$(XBUILT_SOURCES),) + XBUILT_SOURCES := $$(XBUILT_SOURCES) $$(XTARGET) +else + XBUILT_SOURCES := $$(XTARGET) +endif +$$(XTARGET) : $(2) + $$(THRIFT) -o $1 $3 $$< +endef + +clean-common: + rm -rf gen-* diff --git a/contrib/fb303/if/fb303.thrift b/contrib/fb303/if/fb303.thrift new file mode 100644 index 00000000..66c83152 --- /dev/null +++ b/contrib/fb303/if/fb303.thrift @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * fb303.thrift + */ + +namespace java com.facebook.fb303 +namespace cpp facebook.fb303 +namespace perl Facebook.FB303 + +/** + * Common status reporting mechanism across all services + */ +enum fb_status { + DEAD = 0, + STARTING = 1, + ALIVE = 2, + STOPPING = 3, + STOPPED = 4, + WARNING = 5, +} + +/** + * Standard base service + */ +service FacebookService { + + /** + * Returns a descriptive name of the service + */ + string getName(), + + /** + * Returns the version of the service + */ + string getVersion(), + + /** + * Gets the status of this service + */ + fb_status getStatus(), + + /** + * User friendly description of status, such as why the service is in + * the dead or warning state, or what is being started or stopped. + */ + string getStatusDetails(), + + /** + * Gets the counters for this service + */ + map getCounters(), + + /** + * Gets the value of a single counter + */ + i64 getCounter(1: string key), + + /** + * Sets an option + */ + void setOption(1: string key, 2: string value), + + /** + * Gets an option + */ + string getOption(1: string key), + + /** + * Gets all options + */ + map getOptions(), + + /** + * Returns a CPU profile over the given time interval (client and server + * must agree on the profile format). + */ + string getCpuProfile(1: i32 profileDurationInSec), + + /** + * Returns the unix time that the server has been running since + */ + i64 aliveSince(), + + /** + * Tell the server to reload its configuration, reopen log files, etc + */ + oneway void reinitialize(), + + /** + * Suggest a shutdown to the server + */ + oneway void shutdown(), + +} diff --git a/contrib/fb303/java/FacebookBase.java b/contrib/fb303/java/FacebookBase.java new file mode 100644 index 00000000..5778cc8b --- /dev/null +++ b/contrib/fb303/java/FacebookBase.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.fb303; + +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.concurrent.ConcurrentHashMap; + +public abstract class FacebookBase implements FacebookService.Iface { + + private String name_; + + private long alive_; + + private final ConcurrentHashMap counters_ = + new ConcurrentHashMap(); + + private final ConcurrentHashMap options_ = + new ConcurrentHashMap(); + + protected FacebookBase(String name) { + name_ = name; + alive_ = System.currentTimeMillis() / 1000; + } + + public String getName() { + return name_; + } + + public abstract int getStatus(); + + public String getStatusDetails() { + return ""; + } + + public void deleteCounter(String key) { + counters_.remove(key); + } + + public void resetCounter(String key) { + counters_.put(key, 0L); + } + + public long incrementCounter(String key) { + long val = getCounter(key) + 1; + counters_.put(key, val); + return val; + } + + public AbstractMap getCounters() { + return counters_; + } + + public long getCounter(String key) { + Long val = counters_.get(key); + if (val == null) { + return 0; + } + return val.longValue(); + } + + public void setOption(String key, String value) { + options_.put(key, value); + } + + public String getOption(String key) { + return options_.get(key); + } + + public AbstractMap getOptions() { + return options_; + } + + public long aliveSince() { + return alive_; + } + + public String getCpuProfile() { + return ""; + } + + public void reinitialize() {} + + public void shutdown() {} + +} diff --git a/contrib/fb303/java/build.xml b/contrib/fb303/java/build.xml new file mode 100755 index 00000000..4ad30e53 --- /dev/null +++ b/contrib/fb303/java/build.xml @@ -0,0 +1,84 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + generating thrift fb303 files + + + + + + + + + + + + + Building libfb303.jar .... + + + + + + + + + + + + + + + + + + + + Cleaning old stuff .... + + + + + diff --git a/contrib/fb303/php/FacebookBase.php b/contrib/fb303/php/FacebookBase.php new file mode 100644 index 00000000..2ac318fb --- /dev/null +++ b/contrib/fb303/php/FacebookBase.php @@ -0,0 +1,89 @@ +name_ = $name; + } + + public function getName() { + return $this->name_; + } + + public function getVersion() { + return ''; + } + + public function getStatus() { + return null; + } + + public function getStatusDetails() { + return ''; + } + + public function getCounters() { + return array(); + } + + public function getCounter($key) { + return null; + } + + public function setOption($key, $value) { + return; + } + + public function getOption($key) { + return ''; + } + + public function getOptions() { + return array(); + } + + public function aliveSince() { + return 0; + } + + public function getCpuProfile($duration) { + return ''; + } + + public function getLimitedReflection() { + return array(); + } + + public function reinitialize() { + return; + } + + public function shutdown() { + return; + } + +} + diff --git a/contrib/fb303/py/Makefile.am b/contrib/fb303/py/Makefile.am new file mode 100644 index 00000000..060495e5 --- /dev/null +++ b/contrib/fb303/py/Makefile.am @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +DESTDIR ?= / +EXTRA_DIST = setup.py src + +all: + +all-local: + $(thrift_home)/bin/thrift --gen py $(top_srcdir)/if/fb303.thrift + mv gen-py/fb303/* fb303 + $(PYTHON) setup.py build + +# We're ignoring prefix here because site-packages seems to be +# the equivalent of /usr/local/lib in Python land. +# Old version (can't put inline because it's not portable). +#$(PYTHON) setup.py install --prefix=$(prefix) --root=$(DESTDIR) $(PYTHON_SETUPUTIL_ARGS) +install-exec-hook: + $(PYTHON) setup.py install --root=$(DESTDIR) --prefix=$(PY_PREFIX) $(PYTHON_SETUPUTIL_ARGS) + + + +clean: clean-local + +clean-local: + $(RM) -r build + +check-local: all diff --git a/contrib/fb303/py/fb303/FacebookBase.py b/contrib/fb303/py/fb303/FacebookBase.py new file mode 100644 index 00000000..685ff20f --- /dev/null +++ b/contrib/fb303/py/fb303/FacebookBase.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import time +import FacebookService +import thrift.reflection.limited +from ttypes import fb_status + +class FacebookBase(FacebookService.Iface): + + def __init__(self, name): + self.name = name + self.alive = int(time.time()) + self.counters = {} + + def getName(self, ): + return self.name + + def getVersion(self, ): + return '' + + def getStatus(self, ): + return fb_status.ALIVE + + def getCounters(self): + return self.counters + + def resetCounter(self, key): + self.counters[key] = 0 + + def getCounter(self, key): + if self.counters.has_key(key): + return self.counters[key] + return 0 + + def incrementCounter(self, key): + self.counters[key] = self.getCounter(key) + 1 + + def setOption(self, key, value): + pass + + def getOption(self, key): + return "" + + def getOptions(self): + return {} + + def getOptions(self): + return {} + + def aliveSince(self): + return self.alive + + def getCpuProfile(self, duration): + return "" + + def getLimitedReflection(self): + return thrift.reflection.limited.Service() + + def reinitialize(self): + pass + + def shutdown(self): + pass diff --git a/contrib/fb303/py/fb303_scripts/__init__.py b/contrib/fb303/py/fb303_scripts/__init__.py new file mode 100644 index 00000000..f8e3a94b --- /dev/null +++ b/contrib/fb303/py/fb303_scripts/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['fb303_simple_mgmt'] diff --git a/contrib/fb303/py/fb303_scripts/fb303_simple_mgmt.py b/contrib/fb303/py/fb303_scripts/fb303_simple_mgmt.py new file mode 100644 index 00000000..4f8ce993 --- /dev/null +++ b/contrib/fb303/py/fb303_scripts/fb303_simple_mgmt.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys, os +from optparse import OptionParser + +from thrift.Thrift import * + +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol + +from fb303 import * +from fb303.ttypes import * + +def service_ctrl( + command, + port, + trans_factory = None, + prot_factory = None): + """ + service_ctrl is a generic function to execute standard fb303 functions + + @param command: one of stop, start, reload, status, counters, name, alive + @param port: service's port + @param trans_factory: TTransportFactory to use for obtaining a TTransport. Default is + TBufferedTransportFactory + @param prot_factory: TProtocolFactory to use for obtaining a TProtocol. Default is + TBinaryProtocolFactory + """ + + if command in ["status"]: + try: + status = fb303_wrapper('status', port, trans_factory, prot_factory) + status_details = fb303_wrapper('get_status_details', port, trans_factory, prot_factory) + + msg = fb_status_string(status) + if (len(status_details)): + msg += " - %s" % status_details + print msg + + if (status == fb_status.ALIVE): + return 2 + else: + return 3 + except: + print "Failed to get status" + return 3 + + # scalar commands + if command in ["version","alive","name"]: + try: + result = fb303_wrapper(command, port, trans_factory, prot_factory) + print result + return 0 + except: + print "failed to get ",command + return 3 + + # counters + if command in ["counters"]: + try: + counters = fb303_wrapper('counters', port, trans_factory, prot_factory) + for counter in counters: + print "%s: %d" % (counter, counters[counter]) + return 0 + except: + print "failed to get counters" + return 3 + + + # Only root should be able to run the following commands + if os.getuid() == 0: + # async commands + if command in ["stop","reload"] : + try: + fb303_wrapper(command, port, trans_factory, prot_factory) + return 0 + except: + print "failed to tell the service to ", command + return 3 + else: + if command in ["stop","reload"]: + print "root privileges are required to stop or reload the service." + return 4 + + print "The following commands are available:" + for command in ["counters","name","version","alive","status"]: + print "\t%s" % command + print "The following commands are available for users with root privileges:" + for command in ["stop","reload"]: + print "\t%s" % command + + + + return 0; + + +def fb303_wrapper(command, port, trans_factory = None, prot_factory = None): + sock = TSocket.TSocket('localhost', port) + + # use input transport factory if provided + if (trans_factory is None): + trans = TTransport.TBufferedTransport(sock) + else: + trans = trans_factory.getTransport(sock) + + # use input protocol factory if provided + if (prot_factory is None): + prot = TBinaryProtocol.TBinaryProtocol(trans) + else: + prot = prot_factory.getProtocol(trans) + + # initialize client and open transport + fb303_client = FacebookService.Client(prot, prot) + trans.open() + + if (command == 'reload'): + fb303_client.reinitialize() + + elif (command == 'stop'): + fb303_client.shutdown() + + elif (command == 'status'): + return fb303_client.getStatus() + + elif (command == 'version'): + return fb303_client.getVersion() + + elif (command == 'get_status_details'): + return fb303_client.getStatusDetails() + + elif (command == 'counters'): + return fb303_client.getCounters() + + elif (command == 'name'): + return fb303_client.getName() + + elif (command == 'alive'): + return fb303_client.aliveSince() + + trans.close() + + +def fb_status_string(status_enum): + if (status_enum == fb_status.DEAD): + return "DEAD" + if (status_enum == fb_status.STARTING): + return "STARTING" + if (status_enum == fb_status.ALIVE): + return "ALIVE" + if (status_enum == fb_status.STOPPING): + return "STOPPING" + if (status_enum == fb_status.STOPPED): + return "STOPPED" + if (status_enum == fb_status.WARNING): + return "WARNING" + + +def main(): + + # parse command line options + parser = OptionParser() + commands=["stop","counters","status","reload","version","name","alive"] + + parser.add_option("-c", "--command", dest="command", help="execute this API", + choices=commands, default="status") + parser.add_option("-p","--port",dest="port",help="the service's port", + default=9082) + + (options, args) = parser.parse_args() + status = service_ctrl(options.command, options.port) + sys.exit(status) + + +if __name__ == '__main__': + main() diff --git a/contrib/fb303/py/setup.py b/contrib/fb303/py/setup.py new file mode 100644 index 00000000..a29f9642 --- /dev/null +++ b/contrib/fb303/py/setup.py @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from distutils.core import setup + +setup(name='fb303', + version='1.0', + packages=['fb303', 'fb303_scripts'], + ) + + diff --git a/contrib/thrift.el b/contrib/thrift.el new file mode 100644 index 00000000..cd3e0e89 --- /dev/null +++ b/contrib/thrift.el @@ -0,0 +1,126 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one +;; or more contributor license agreements. See the NOTICE file +;; distributed with this work for additional information +;; regarding copyright ownership. The ASF licenses this file +;; to you under the Apache License, Version 2.0 (the +;; "License"); you may not use this file except in compliance +;; with the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, +;; software distributed under the License is distributed on an +;; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +;; KIND, either express or implied. See the License for the +;; specific language governing permissions and limitations +;; under the License. +;; + +(require 'font-lock) + +(defvar thrift-mode-hook nil) +(add-to-list 'auto-mode-alist '("\\.thrift\\'" . thrift-mode)) + +(defvar thrift-indent-level 2 + "Defines 2 spaces for thrift indentation.") + +;; syntax coloring +(defconst thrift-font-lock-keywords + (list + '("#.*$" . font-lock-comment-face) ;; perl style comments + '("\\<\\(include\\|struct\\|exception\\|typedef\\|const\\|enum\\|service\\|extends\\|void\\|oneway\\|throws\\|optional\\|required\\)\\>" . font-lock-keyword-face) ;; keywords + '("\\<\\(bool\\|byte\\|i16\\|i32\\|i64\\|double\\|string\\|binary\\|map\\|list\\|set\\)\\>" . font-lock-type-face) ;; built-in types + '("\\<\\([0-9]+\\)\\>" . font-lock-variable-name-face) ;; ordinals + '("\\<\\(\\w+\\)\\s-*(" (1 font-lock-function-name-face)) ;; functions + ) + "Thrift Keywords") + +;; indentation +(defun thrift-indent-line () + "Indent current line as Thrift code." + (interactive) + (beginning-of-line) + (if (bobp) + (indent-line-to 0) + (let ((not-indented t) cur-indent) + (if (looking-at "^[ \t]*\\(}\\|throws\\)") + (if (looking-at "^[ \t]*}") + (progn + (save-excursion + (forward-line -1) + (setq cur-indent (- (current-indentation) thrift-indent-level))) + (if (< cur-indent 0) + (setq cur-indent 0))) + (progn + (save-excursion + (forward-line -1) + (if (looking-at "^[ \t]*[\\.<>[:word:]]+[ \t]+[\\.<>[:word:]]+[ \t]*(") + (setq cur-indent (+ (current-indentation) thrift-indent-level)) + (setq cur-indent (current-indentation)))))) + (save-excursion + (while not-indented + (forward-line -1) + (if (looking-at "^[ \t]*}") + (progn + (setq cur-indent (current-indentation)) + (setq not-indented nil)) + (if (looking-at "^.*{[^}]*$") + (progn + (setq cur-indent (+ (current-indentation) thrift-indent-level)) + (setq not-indented nil)) + (if (bobp) + (setq not-indented nil))) + (if (looking-at "^[ \t]*throws") + (progn + (setq cur-indent (- (current-indentation) thrift-indent-level)) + (if (< cur-indent 0) + (setq cur-indent 0)) + (setq not-indented nil)) + (if (bobp) + (setq not-indented nil))) + (if (looking-at "^[ \t]*[\\.<>[:word:]]+[ \t]+[\\.<>[:word:]]+[ \t]*([^)]*$") + (progn + (setq cur-indent (+ (current-indentation) thrift-indent-level)) + (setq not-indented nil)) + (if (bobp) + (setq not-indented nil))) + (if (looking-at "^[ \t]*\\/\\*") + (progn + (setq cur-indent (+ (current-indentation) 1)) + (setq not-indented nil)) + (if (bobp) + (setq not-indented nil))) + (if (looking-at "^[ \t]*\\*\\/") + (progn + (setq cur-indent (- (current-indentation) 1)) + (setq not-indented nil)) + (if (bobp) + (setq not-indented nil))) + )))) + (if cur-indent + (indent-line-to cur-indent) + (indent-line-to 0))))) + +;; C/C++ comments; also allowing underscore in words +(defvar thrift-mode-syntax-table + (let ((thrift-mode-syntax-table (make-syntax-table))) + (modify-syntax-entry ?_ "w" thrift-mode-syntax-table) + (modify-syntax-entry ?/ ". 1456" thrift-mode-syntax-table) + (modify-syntax-entry ?* ". 23" thrift-mode-syntax-table) + (modify-syntax-entry ?\n "> b" thrift-mode-syntax-table) + thrift-mode-syntax-table) + "Syntax table for thrift-mode") + +(defun thrift-mode () + "Mode for editing Thrift files" + (interactive) + (kill-all-local-variables) + (set-syntax-table thrift-mode-syntax-table) + (set (make-local-variable 'font-lock-defaults) '(thrift-font-lock-keywords)) + (setq major-mode 'thrift-mode) + (setq mode-name "Thrift") + (run-hooks 'thrift-mode-hook) + (set (make-local-variable 'indent-line-function) 'thrift-indent-line) + ) +(provide 'thrift-mode) diff --git a/contrib/thrift.spec b/contrib/thrift.spec new file mode 100644 index 00000000..ecee631a --- /dev/null +++ b/contrib/thrift.spec @@ -0,0 +1,206 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# TODO(dreiss): Have a Python build with and without the extension. +%{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +%{!?python_sitearch: %define python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)")} +# TODO(dreiss): Where is this supposed to go? +%{!?thrift_erlang_root: %define thrift_erlang_root /opt/thrift-erl} + +Name: thrift +License: Apache License v2.0 +Group: Development +Summary: RPC and serialization framework +Version: 20080529svn +Epoch: 1 +Release: 1 +URL: http://developers.facebook.com/thrift +Packager: David Reiss +Source0: %{name}-%{version}.tar.gz + +BuildRequires: gcc >= 3.4.6 +BuildRequires: gcc-c++ + +# TODO(dreiss): Can these be moved into the individual packages? +%if %{!?without_java: 1} +BuildRequires: java-devel >= 0:1.5.0 +BuildRequires: ant >= 0:1.6.5 +%endif + +%if %{!?without_python: 1} +BuildRequires: python-devel +%endif + +%if %{!?without_erlang: 1} +BuildRequires: erlang +%endif + +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) + +%description +Thrift is a software framework for scalable cross-language services +development. It combines a powerful software stack with a code generation +engine to build services that work efficiently and seamlessly between C++, +Java, C#, Python, Ruby, Perl, PHP, Objective C/Cocoa, Smalltalk, Erlang, +Objective Caml, and Haskell. + +%files +%defattr(-,root,root) +%{_bindir}/thrift + + +%package lib-cpp +Summary: Thrift C++ library +Group: Libraries + +%description lib-cpp +C++ libraries for Thrift. + +%files lib-cpp +%defattr(-,root,root) +%{_libdir}/libthrift*.so.* + + +%package lib-cpp-devel +Summary: Thrift C++ library development files +Group: Libraries +Requires: %{name} = %{version}-%{release} +Requires: boost-devel +%if %{!?without_libevent: 1} +Requires: libevent-devel >= 1.2 +%endif +%if %{!?without_zlib: 1} +Requires: zlib-devel +%endif + +%description lib-cpp-devel +C++ static libraries and headers for Thrift. + +%files lib-cpp-devel +%defattr(-,root,root) +%{_includedir}/thrift/ +%{_libdir}/libthrift*.*a +%{_libdir}/libthrift*.so +%{_libdir}/pkgconfig/thrift*.pc + + +%if %{!?without_java: 1} +%package lib-java +Summary: Thrift Java library +Group: Libraries +Requires: java >= 0:1.5.0 + +%description lib-java +Java libraries for Thrift. + +%files lib-java +%defattr(-,root,root) +%{_javadir}/* +%endif + + +%if %{!?without_python: 1} +%package lib-python +Summary: Thrift Python library +Group: Libraries + +%description lib-python +Python libraries for Thrift. + +%files lib-python +%defattr(-,root,root) +%{python_sitearch}/* +%endif + + +%if %{!?without_erlang: 1} +%package lib-erlang +Summary: Thrift Python library +Group: Libraries +Requires: erlang + +%description lib-erlang +Erlang libraries for Thrift. + +%files lib-erlang +%defattr(-,root,root) +%{thrift_erlang_root} +%endif + + +%prep +%setup -q + +%build +# TODO(dreiss): Implement a single --without-build-kludges. +%configure \ + %{?without_libevent: --without-libevent } \ + %{?without_zlib: --without-zlib } \ + --without-java \ + --without-csharp \ + --without-py \ + --without-erlang \ + +make + +%if %{!?without_java: 1} +cd lib/java +%ant +cd ../.. +%endif + +%if %{!?without_python: 1} +cd lib/py +CFLAGS="%{optflags}" %{__python} setup.py build +cd ../.. +%endif + +%if %{!?without_erlang: 1} +cd lib/erl +make +cd ../.. +%endif + +%install +%makeinstall + +%if %{!?without_java: 1} +mkdir -p $RPM_BUILD_ROOT%{_javadir} +cp -p lib/java/*.jar $RPM_BUILD_ROOT%{_javadir} +%endif + +%if %{!?without_python: 1} +cd lib/py +%{__python} setup.py install -O1 --skip-build --root $RPM_BUILD_ROOT +cd ../.. +%endif + +%if %{!?without_erlang: 1} +mkdir -p ${RPM_BUILD_ROOT}%{thrift_erlang_root} +cp -r lib/erl/ebin ${RPM_BUILD_ROOT}%{thrift_erlang_root} +%endif + + +%clean +rm -rf ${RPM_BUILD_ROOT} + + +%changelog +* Wed May 28 2008 David Reiss - 20080529svn +- Initial build, based on the work of Kevin Smith and Ben Maurer. diff --git a/contrib/thrift.vim b/contrib/thrift.vim new file mode 100644 index 00000000..79ce5472 --- /dev/null +++ b/contrib/thrift.vim @@ -0,0 +1,91 @@ +" Vim syntax file +" Language: Thrift +" Maintainer: Martin Smith +" Last Change: $Date: $ +" Copy to ~/.vim/ +" Add to ~/.vimrc +" au BufRead,BufNewFile *.thrift set filetype=thrift +" au! Syntax thrift source ~/.vim/thrift.vim +" +" $Id: $ +" +" Licensed to the Apache Software Foundation (ASF) under one +" or more contributor license agreements. See the NOTICE file +" distributed with this work for additional information +" regarding copyright ownership. The ASF licenses this file +" to you under the Apache License, Version 2.0 (the +" "License"); you may not use this file except in compliance +" with the License. You may obtain a copy of the License at +" +" http://www.apache.org/licenses/LICENSE-2.0 +" +" Unless required by applicable law or agreed to in writing, +" software distributed under the License is distributed on an +" "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +" KIND, either express or implied. See the License for the +" specific language governing permissions and limitations +" under the License. +" + +if version < 600 + syntax clear +elseif exists("b:current_syntax") + finish +endif + +" Todo +syn keyword thriftTodo TODO todo FIXME fixme XXX xxx contained + +" Comments +syn match thriftComment "#.*" contains=thriftTodo +syn region thriftComment start="/\*" end="\*/" contains=thriftTodo +syn match thriftComment "//.\{-}\(?>\|$\)\@=" + +" String +syn region thriftStringDouble matchgroup=None start=+"+ end=+"+ + +" Number +syn match thriftNumber "-\=\<\d\+\>" contained + +" Keywords +syn keyword thriftKeyword namespace +syn keyword thriftKeyword xsd_all xsd_optional xsd_nillable xsd_attrs +syn keyword thriftKeyword include cpp_include cpp_type const optional required +syn keyword thriftBasicTypes void bool byte i16 i32 i64 double string binary +syn keyword thriftStructure map list set struct typedef exception enum throws + +" Special +syn match thriftSpecial "\d\+:" + +" Structure +syn keyword thriftStructure service oneway extends +"async" { return tok_async; } +"exception" { return tok_xception; } +"extends" { return tok_extends; } +"throws" { return tok_throws; } +"service" { return tok_service; } +"enum" { return tok_enum; } +"const" { return tok_const; } + +if version >= 508 || !exists("did_thrift_syn_inits") + if version < 508 + let did_thrift_syn_inits = 1 + command! -nargs=+ HiLink hi link + else + command! -nargs=+ HiLink hi def link + endif + + HiLink thriftComment Comment + HiLink thriftKeyword Special + HiLink thriftBasicTypes Type + HiLink thriftStructure StorageClass + HiLink thriftTodo Todo + HiLink thriftString String + HiLink thriftNumber Number + HiLink thriftSpecial Special + HiLink thriftStructure Structure + + delcommand HiLink +endif + +let b:current_syntax = "thrift" diff --git a/contrib/thrift_dump.cpp b/contrib/thrift_dump.cpp new file mode 100644 index 00000000..0ddfcec3 --- /dev/null +++ b/contrib/thrift_dump.cpp @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include + +using namespace std; +using boost::shared_ptr; +using namespace apache::thrift::transport; +using namespace apache::thrift::protocol; + +void usage() { + fprintf(stderr, + "usage: thrift_dump {-b|-f|-s} < input > ouput\n" + " -b TBufferedTransport messages\n" + " -f TFramedTransport messages\n" + " -s Raw structures\n"); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) { + if (argc != 2) { + usage(); + } + + shared_ptr stdin_trans(new TFDTransport(STDIN_FILENO)); + shared_ptr itrans; + + if (argv[1] == std::string("-b") || argv[1] == std::string("-s")) { + itrans.reset(new TBufferedTransport(stdin_trans)); + } else if (argv[1] == std::string("-f")) { + itrans.reset(new TFramedTransport(stdin_trans)); + } else { + usage(); + } + + shared_ptr iprot(new TBinaryProtocol(itrans)); + shared_ptr oprot( + new TDebugProtocol( + shared_ptr(new TBufferedTransport( + shared_ptr(new TFDTransport(STDOUT_FILENO)))))); + + TProtocolTap tap(iprot, oprot); + + try { + if (argv[1] == std::string("-s")) { + for (;;) { + tap.skip(T_STRUCT); + } + } else { + std::string name; + TMessageType messageType; + int32_t seqid; + for (;;) { + tap.readMessageBegin(name, messageType, seqid); + tap.skip(T_STRUCT); + tap.readMessageEnd(); + } + } + } catch (TProtocolException exn) { + cout << "Protocol Exception: " << exn.what() << endl; + } catch (...) { + oprot->getTransport()->flush(); + } + + cout << endl; + + return 0; +} diff --git a/doc/lgpl-2.1.txt b/doc/lgpl-2.1.txt new file mode 100644 index 00000000..5ab7695a --- /dev/null +++ b/doc/lgpl-2.1.txt @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/doc/otp-base-license.txt b/doc/otp-base-license.txt new file mode 100644 index 00000000..8ee29920 --- /dev/null +++ b/doc/otp-base-license.txt @@ -0,0 +1,20 @@ +Tue Oct 24 12:28:44 CDT 2006 + +Copyright (c) <2006> + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software (OTP Base, fslib, G.A.S) and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/doc/thrift.bnf b/doc/thrift.bnf new file mode 100644 index 00000000..24d83f68 --- /dev/null +++ b/doc/thrift.bnf @@ -0,0 +1,96 @@ +Thrift Protocol Structure + +Last Modified: 2007-Jun-29 + +-------------------------------------------------------------------- + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +-------------------------------------------------------------------- + +This document describes the structure of the Thrift protocol +without specifying the encoding. Thus, the order of elements +could in some cases be rearranged depending upon the TProtocol +implementation, but this document specifies the minimum required +structure. There are some "dumb" terminals like STRING and INT +that take the place of an actual encoding specification. + +They key point to notice is that ALL messages are just one wrapped +. Depending upon the message type, the can be +interpreted as the argument list to a function, the return value +of a function, or an exception. + +-------------------------------------------------------------------- + + ::= + + ::= + + ::= STRING + + ::= T_CALL | T_REPLY | T_EXCEPTION + + ::= I32 + + ::= * + + ::= + + ::= STRING + + ::= T_STOP + + ::= + + ::= + + ::= STRING + + ::= T_BOOL | T_BYTE | T_I8 | T_I16 | T_I32 | T_I64 | T_DOUBLE + | T_STRING | T_BINARY | T_STRUCT | T_MAP | T_SET | T_LIST + + ::= I16 + + ::= I8 | I16 | I32 | I64 | DOUBLE | STRING | BINARY + | | | + + ::= * + + ::= + + ::= + + ::= + + ::= I32 + + ::= * + + ::= + + ::= + + ::= I32 + + ::= * + + ::= + + ::= + + ::= I32 diff --git a/doc/thrift.tex b/doc/thrift.tex new file mode 100644 index 00000000..d20b7377 --- /dev/null +++ b/doc/thrift.tex @@ -0,0 +1,1057 @@ +%----------------------------------------------------------------------------- +% +% Thrift whitepaper +% +% Name: thrift.tex +% +% Authors: Mark Slee (mcslee@facebook.com) +% +% Created: 05 March 2007 +% +% You will need a copy of sigplanconf.cls to format this document. +% It is available at . +% +%----------------------------------------------------------------------------- + + +\documentclass[nocopyrightspace,blockstyle]{sigplanconf} + +\usepackage{amssymb} +\usepackage{amsfonts} +\usepackage{amsmath} +\usepackage{url} + +\begin{document} + +% \conferenceinfo{WXYZ '05}{date, City.} +% \copyrightyear{2007} +% \copyrightdata{[to be supplied]} + +% \titlebanner{banner above paper title} % These are ignored unless +% \preprintfooter{short description of paper} % 'preprint' option specified. + +\title{Thrift: Scalable Cross-Language Services Implementation} +\subtitle{} + +\authorinfo{Mark Slee, Aditya Agarwal and Marc Kwiatkowski} + {Facebook, 156 University Ave, Palo Alto, CA} + {\{mcslee,aditya,marc\}@facebook.com} + +\maketitle + +\begin{abstract} +Thrift is a software library and set of code-generation tools developed at +Facebook to expedite development and implementation of efficient and scalable +backend services. Its primary goal is to enable efficient and reliable +communication across programming languages by abstracting the portions of each +language that tend to require the most customization into a common library +that is implemented in each language. Specifically, Thrift allows developers to +define datatypes and service interfaces in a single language-neutral file +and generate all the necessary code to build RPC clients and servers. + +This paper details the motivations and design choices we made in Thrift, as +well as some of the more interesting implementation details. It is not +intended to be taken as research, but rather it is an exposition on what we did +and why. +\end{abstract} + +% \category{D.3.3}{Programming Languages}{Language constructs and features} + +%\terms +%Languages, serialization, remote procedure call + +%\keywords +%Data description language, interface definition language, remote procedure call + +\section{Introduction} +As Facebook's traffic and network structure have scaled, the resource +demands of many operations on the site (i.e. search, +ad selection and delivery, event logging) have presented technical requirements +drastically outside the scope of the LAMP framework. In our implementation of +these services, various programming languages have been selected to +optimize for the right combination of performance, ease and speed of +development, availability of existing libraries, etc. By and large, +Facebook's engineering culture has tended towards choosing the best +tools and implementations available over standardizing on any one +programming language and begrudgingly accepting its inherent limitations. + +Given this design choice, we were presented with the challenge of building +a transparent, high-performance bridge across many programming languages. +We found that most available solutions were either too limited, did not offer +sufficient datatype freedom, or suffered from subpar performance. +\footnote{See Appendix A for a discussion of alternative systems.} + +The solution that we have implemented combines a language-neutral software +stack implemented across numerous programming languages and an associated code +generation engine that transforms a simple interface and data definition +language into client and server remote procedure call libraries. +Choosing static code generation over a dynamic system allows us to create +validated code that can be run without the need for +any advanced introspective run-time type checking. It is also designed to +be as simple as possible for the developer, who can typically define all +the necessary data structures and interfaces for a complex service in a single +short file. + +Surprised that a robust open solution to these relatively common problems +did not yet exist, we committed early on to making the Thrift implementation +open source. + +In evaluating the challenges of cross-language interaction in a networked +environment, some key components were identified: + +\textit{Types.} A common type system must exist across programming languages +without requiring that the application developer use custom Thrift datatypes +or write their own serialization code. That is, +a C++ programmer should be able to transparently exchange a strongly typed +STL map for a dynamic Python dictionary. Neither +programmer should be forced to write any code below the application layer +to achieve this. Section 2 details the Thrift type system. + +\textit{Transport.} Each language must have a common interface to +bidirectional raw data transport. The specifics of how a given +transport is implemented should not matter to the service developer. +The same application code should be able to run against TCP stream sockets, +raw data in memory, or files on disk. Section 3 details the Thrift Transport +layer. + +\textit{Protocol.} Datatypes must have some way of using the Transport +layer to encode and decode themselves. Again, the application +developer need not be concerned by this layer. Whether the service uses +an XML or binary protocol is immaterial to the application code. +All that matters is that the data can be read and written in a consistent, +deterministic matter. Section 4 details the Thrift Protocol layer. + +\textit{Versioning.} For robust services, the involved datatypes must +provide a mechanism for versioning themselves. Specifically, +it should be possible to add or remove fields in an object or alter the +argument list of a function without any interruption in service (or, +worse yet, nasty segmentation faults). Section 5 details Thrift's versioning +system. + +\textit{Processors.} Finally, we generate code capable of processing data +streams to accomplish remote procedure calls. Section 6 details the generated +code and TProcessor paradigm. + +Section 7 discusses implementation details, and Section 8 describes +our conclusions. + +\section{Types} + +The goal of the Thrift type system is to enable programmers to develop using +completely natively defined types, no matter what programming language they +use. By design, the Thrift type system does not introduce any special dynamic +types or wrapper objects. It also does not require that the developer write +any code for object serialization or transport. The Thrift IDL (Interface +Definition Language) file is +logically a way for developers to annotate their data structures with the +minimal amount of extra information necessary to tell a code generator +how to safely transport the objects across languages. + +\subsection{Base Types} + +The type system rests upon a few base types. In considering which types to +support, we aimed for clarity and simplicity over abundance, focusing +on the key types available in all programming languages, ommitting any +niche types available only in specific languages. + +The base types supported by Thrift are: +\begin{itemize} +\item \texttt{bool} A boolean value, true or false +\item \texttt{byte} A signed byte +\item \texttt{i16} A 16-bit signed integer +\item \texttt{i32} A 32-bit signed integer +\item \texttt{i64} A 64-bit signed integer +\item \texttt{double} A 64-bit floating point number +\item \texttt{string} An encoding-agnostic text or binary string +\item \texttt{binary} A byte array representation for blobs +\end{itemize} + +Of particular note is the absence of unsigned integer types. Because these +types have no direct translation to native primitive types in many languages, +the advantages they afford are lost. Further, there is no way to prevent the +application developer in a language like Python from assigning a negative value +to an integer variable, leading to unpredictable behavior. From a design +standpoint, we observed that unsigned integers were very rarely, if ever, used +for arithmetic purposes, but in practice were much more often used as keys or +identifiers. In this case, the sign is irrelevant. Signed integers serve this +same purpose and can be safely cast to their unsigned counterparts (most +commonly in C++) when absolutely necessary. + +\subsection{Structs} + +A Thrift struct defines a common object to be used across languages. A struct +is essentially equivalent to a class in object oriented programming +languages. A struct has a set of strongly typed fields, each with a unique +name identifier. The basic syntax for defining a Thrift struct looks very +similar to a C struct definition. Fields may be annotated with an integer field +identifier (unique to the scope of that struct) and optional default values. +Field identifiers will be automatically assigned if omitted, though they are +strongly encouraged for versioning reasons discussed later. + +\subsection{Containers} + +Thrift containers are strongly typed containers that map to the most commonly +used containers in common programming languages. They are annotated using +the C++ template (or Java Generics) style. There are three types available: +\begin{itemize} +\item \texttt{list} An ordered list of elements. Translates directly into +an STL \texttt{vector}, Java \texttt{ArrayList}, or native array in scripting languages. May +contain duplicates. +\item \texttt{set} An unordered set of unique elements. Translates into +an STL \texttt{set}, Java \texttt{HashSet}, \texttt{set} in Python, or native +dictionary in PHP/Ruby. +\item \texttt{map} A map of strictly unique keys to values +Translates into an STL \texttt{map}, Java \texttt{HashMap}, PHP associative +array, or Python/Ruby dictionary. +\end{itemize} + +While defaults are provided, the type mappings are not explicitly fixed. Custom +code generator directives have been added to substitute custom types in +destination languages (i.e. +\texttt{hash\_map} or Google's sparse hash map can be used in C++). The +only requirement is that the custom types support all the necessary iteration +primitives. Container elements may be of any valid Thrift type, including other +containers or structs. + +\begin{verbatim} +struct Example { + 1:i32 number=10, + 2:i64 bigNumber, + 3:double decimals, + 4:string name="thrifty" +}\end{verbatim} + +In the target language, each definition generates a type with two methods, +\texttt{read} and \texttt{write}, which perform serialization and transport +of the objects using a Thrift TProtocol object. + +\subsection{Exceptions} + +Exceptions are syntactically and functionally equivalent to structs except +that they are declared using the \texttt{exception} keyword instead of the +\texttt{struct} keyword. + +The generated objects inherit from an exception base class as appropriate +in each target programming language, in order to seamlessly +integrate with native exception handling in any given +language. Again, the design emphasis is on making the code familiar to the +application developer. + +\subsection{Services} + +Services are defined using Thrift types. Definition of a service is +semantically equivalent to defining an interface (or a pure virtual abstract +class) in object oriented +programming. The Thrift compiler generates fully functional client and +server stubs that implement the interface. Services are defined as follows: + +\begin{verbatim} +service { + () + [throws ()] + ... +}\end{verbatim} + +An example: + +\begin{verbatim} +service StringCache { + void set(1:i32 key, 2:string value), + string get(1:i32 key) throws (1:KeyNotFound knf), + void delete(1:i32 key) +} +\end{verbatim} + +Note that \texttt{void} is a valid type for a function return, in addition to +all other defined Thrift types. Additionally, an \texttt{async} modifier +keyword may be added to a \texttt{void} function, which will generate code that does +not wait for a response from the server. Note that a pure \texttt{void} +function will return a response to the client which guarantees that the +operation has completed on the server side. With \texttt{async} method calls +the client will only be guaranteed that the request succeeded at the +transport layer. (In many transport scenarios this is inherently unreliable +due to the Byzantine Generals' Problem. Therefore, application developers +should take care only to use the async optimization in cases where dropped +method calls are acceptable or the transport is known to be reliable.) + +Also of note is the fact that argument lists and exception lists for functions +are implemented as Thrift structs. All three constructs are identical in both +notation and behavior. + +\section{Transport} + +The transport layer is used by the generated code to facilitate data transfer. + +\subsection{Interface} + +A key design choice in the implementation of Thrift was to decouple the +transport layer from the code generation layer. Though Thrift is typically +used on top of the TCP/IP stack with streaming sockets as the base layer of +communication, there was no compelling reason to build that constraint into +the system. The performance tradeoff incurred by an abstracted I/O layer +(roughly one virtual method lookup / function call per operation) was +immaterial compared to the cost of actual I/O operations (typically invoking +system calls). + +Fundamentally, generated Thrift code only needs to know how to read and +write data. The origin and destination of the data are irrelevant; it may be a +socket, a segment of shared memory, or a file on the local disk. The Thrift +transport interface supports the following methods: + +\begin{itemize} +\item \texttt{open} Opens the tranpsort +\item \texttt{close} Closes the tranport +\item \texttt{isOpen} Indicates whether the transport is open +\item \texttt{read} Reads from the transport +\item \texttt{write} Writes to the transport +\item \texttt{flush} Forces any pending writes +\end{itemize} + +There are a few additional methods not documented here which are used to aid +in batching reads and optionally signaling the completion of a read or +write operation from the generated code. + +In addition to the above +\texttt{TTransport} interface, there is a\\ +\texttt{TServerTransport} interface +used to accept or create primitive transport objects. Its interface is as +follows: + +\begin{itemize} +\item \texttt{open} Opens the transport +\item \texttt{listen} Begins listening for connections +\item \texttt{accept} Returns a new client transport +\item \texttt{close} Closes the transport +\end{itemize} + +\subsection{Implementation} + +The transport interface is designed for simple implementation in any +programming language. New transport mechanisms can be easily defined as needed +by application developers. + +\subsubsection{TSocket} + +The \texttt{TSocket} class is implemented across all target languages. It +provides a common, simple interface to a TCP/IP stream socket. + +\subsubsection{TFileTransport} + +The \texttt{TFileTransport} is an abstraction of an on-disk file to a data +stream. It can be used to write out a set of incoming Thrift requests to a file +on disk. The on-disk data can then be replayed from the log, either for +post-processing or for reproduction and/or simulation of past events. + +\subsubsection{Utilities} + +The Transport interface is designed to support easy extension using common +OOP techniques, such as composition. Some simple utilites include the +\texttt{TBufferedTransport}, which buffers the writes and reads on an +underlying transport, the \texttt{TFramedTransport}, which transmits data with frame +size headers for chunking optimization or nonblocking operation, and the +\texttt{TMemoryBuffer}, which allows reading and writing directly from the heap +or stack memory owned by the process. + +\section{Protocol} + +A second major abstraction in Thrift is the separation of data structure from +transport representation. Thrift enforces a certain messaging structure when +transporting data, but it is agnostic to the protocol encoding in use. That is, +it does not matter whether data is encoded as XML, human-readable ASCII, or a +dense binary format as long as the data supports a fixed set of operations +that allow it to be deterministically read and written by generated code. + +\subsection{Interface} + +The Thrift Protocol interface is very straightforward. It fundamentally +supports two things: 1) bidirectional sequenced messaging, and +2) encoding of base types, containers, and structs. + +\begin{verbatim} +writeMessageBegin(name, type, seq) +writeMessageEnd() +writeStructBegin(name) +writeStructEnd() +writeFieldBegin(name, type, id) +writeFieldEnd() +writeFieldStop() +writeMapBegin(ktype, vtype, size) +writeMapEnd() +writeListBegin(etype, size) +writeListEnd() +writeSetBegin(etype, size) +writeSetEnd() +writeBool(bool) +writeByte(byte) +writeI16(i16) +writeI32(i32) +writeI64(i64) +writeDouble(double) +writeString(string) + +name, type, seq = readMessageBegin() + readMessageEnd() +name = readStructBegin() + readStructEnd() +name, type, id = readFieldBegin() + readFieldEnd() +k, v, size = readMapBegin() + readMapEnd() +etype, size = readListBegin() + readListEnd() +etype, size = readSetBegin() + readSetEnd() +bool = readBool() +byte = readByte() +i16 = readI16() +i32 = readI32() +i64 = readI64() +double = readDouble() +string = readString() +\end{verbatim} + +Note that every \texttt{write} function has exactly one \texttt{read} counterpart, with +the exception of \texttt{writeFieldStop()}. This is a special method +that signals the end of a struct. The procedure for reading a struct is to +\texttt{readFieldBegin()} until the stop field is encountered, and then to +\texttt{readStructEnd()}. The +generated code relies upon this call sequence to ensure that everything written by +a protocol encoder can be read by a matching protocol decoder. Further note +that this set of functions is by design more robust than necessary. +For example, \texttt{writeStructEnd()} is not strictly necessary, as the end of +a struct may be implied by the stop field. This method is a convenience for +verbose protocols in which it is cleaner to separate these calls (e.g. a closing +\texttt{} tag in XML). + +\subsection{Structure} + +Thrift structures are designed to support encoding into a streaming +protocol. The implementation should never need to frame or compute the +entire data length of a structure prior to encoding it. This is critical to +performance in many scenarios. Consider a long list of relatively large +strings. If the protocol interface required reading or writing a list to be an +atomic operation, then the implementation would need to perform a linear pass over the +entire list before encoding any data. However, if the list can be written +as iteration is performed, the corresponding read may begin in parallel, +theoretically offering an end-to-end speedup of $(kN - C)$, where $N$ is the size +of the list, $k$ the cost factor associated with serializing a single +element, and $C$ is fixed offset for the delay between data being written +and becoming available to read. + +Similarly, structs do not encode their data lengths a priori. Instead, they are +encoded as a sequence of fields, with each field having a type specifier and a +unique field identifier. Note that the inclusion of type specifiers allows +the protocol to be safely parsed and decoded without any generated code +or access to the original IDL file. Structs are terminated by a field header +with a special \texttt{STOP} type. Because all the basic types can be read +deterministically, all structs (even those containing other structs) can be +read deterministically. The Thrift protocol is self-delimiting without any +framing and regardless of the encoding format. + +In situations where streaming is unnecessary or framing is advantageous, it +can be very simply added into the transport layer, using the +\texttt{TFramedTransport} abstraction. + +\subsection{Implementation} + +Facebook has implemented and deployed a space-efficient binary protocol which +is used by most backend services. Essentially, it writes all data +in a flat binary format. Integer types are converted to network byte order, +strings are prepended with their byte length, and all message and field headers +are written using the primitive integer serialization constructs. String names +for fields are omitted - when using generated code, field identifiers are +sufficient. + +We decided against some extreme storage optimizations (i.e. packing +small integers into ASCII or using a 7-bit continuation format) for the sake +of simplicity and clarity in the code. These alterations can easily be made +if and when we encounter a performance-critical use case that demands them. + +\section{Versioning} + +Thrift is robust in the face of versioning and data definition changes. This +is critical to enable staged rollouts of changes to deployed services. The +system must be able to support reading of old data from log files, as well as +requests from out-of-date clients to new servers, and vice versa. + +\subsection{Field Identifiers} + +Versioning in Thrift is implemented via field identifiers. The field header +for every member of a struct in Thrift is encoded with a unique field +identifier. The combination of this field identifier and its type specifier +is used to uniquely identify the field. The Thrift definition language +supports automatic assignment of field identifiers, but it is good +programming practice to always explicitly specify field identifiers. +Identifiers are specified as follows: + +\begin{verbatim} +struct Example { + 1:i32 number=10, + 2:i64 bigNumber, + 3:double decimals, + 4:string name="thrifty" +}\end{verbatim} + +To avoid conflicts between manually and automatically assigned identifiers, +fields with identifiers omitted are assigned identifiers +decrementing from -1, and the language only supports the manual assignment of +positive identifiers. + +When data is being deserialized, the generated code can use these identifiers +to properly identify the field and determine whether it aligns with a field in +its definition file. If a field identifier is not recognized, the generated +code can use the type specifier to skip the unknown field without any error. +Again, this is possible due to the fact that all datatypes are self +delimiting. + +Field identifiers can (and should) also be specified in function argument +lists. In fact, argument lists are not only represented as structs on the +backend, but actually share the same code in the compiler frontend. This +allows for version-safe modification of method parameters + +\begin{verbatim} +service StringCache { + void set(1:i32 key, 2:string value), + string get(1:i32 key) throws (1:KeyNotFound knf), + void delete(1:i32 key) +} +\end{verbatim} + +The syntax for specifying field identifiers was chosen to echo their structure. +Structs can be thought of as a dictionary where the identifiers are keys, and +the values are strongly-typed named fields. + +Field identifiers internally use the \texttt{i16} Thrift type. Note, however, +that the \texttt{TProtocol} abstraction may encode identifiers in any format. + +\subsection{Isset} + +When an unexpected field is encountered, it can be safely ignored and +discarded. When an expected field is not found, there must be some way to +signal to the developer that it was not present. This is implemented via an +inner \texttt{isset} structure inside the defined objects. (Isset functionality +is implicit with a \texttt{null} value in PHP, \texttt{None} in Python +and \texttt{nil} in Ruby.) Essentially, +the inner \texttt{isset} object of each Thrift struct contains a boolean value +for each field which denotes whether or not that field is present in the +struct. When a reader receives a struct, it should check for a field being set +before operating directly on it. + +\begin{verbatim} +class Example { + public: + Example() : + number(10), + bigNumber(0), + decimals(0), + name("thrifty") {} + + int32_t number; + int64_t bigNumber; + double decimals; + std::string name; + + struct __isset { + __isset() : + number(false), + bigNumber(false), + decimals(false), + name(false) {} + bool number; + bool bigNumber; + bool decimals; + bool name; + } __isset; +... +} +\end{verbatim} + +\subsection{Case Analysis} + +There are four cases in which version mismatches may occur. + +\begin{enumerate} +\item \textit{Added field, old client, new server.} In this case, the old +client does not send the new field. The new server recognizes that the field +is not set, and implements default behavior for out-of-date requests. +\item \textit{Removed field, old client, new server.} In this case, the old +client sends the removed field. The new server simply ignores it. +\item \textit{Added field, new client, old server.} The new client sends a +field that the old server does not recognize. The old server simply ignores +it and processes as normal. +\item \textit{Removed field, new client, old server.} This is the most +dangerous case, as the old server is unlikely to have suitable default +behavior implemented for the missing field. It is recommended that in this +situation the new server be rolled out prior to the new clients. +\end{enumerate} + +\subsection{Protocol/Transport Versioning} +The \texttt{TProtocol} abstractions are also designed to give protocol +implementations the freedom to version themselves in whatever manner they +see fit. Specifically, any protocol implementation is free to send whatever +it likes in the \texttt{writeMessageBegin()} call. It is entirely up to the +implementor how to handle versioning at the protocol level. The key point is +that protocol encoding changes are safely isolated from interface definition +version changes. + +Note that the exact same is true of the \texttt{TTransport} interface. For +example, if we wished to add some new checksumming or error detection to the +\texttt{TFileTransport}, we could simply add a version header into the +data it writes to the file in such a way that it would still accept old +log files without the given header. + +\section{RPC Implementation} + +\subsection{TProcessor} + +The last core interface in the Thrift design is the \texttt{TProcessor}, +perhaps the most simple of the constructs. The interface is as follows: + +\begin{verbatim} +interface TProcessor { + bool process(TProtocol in, TProtocol out) + throws TException +} +\end{verbatim} + +The key design idea here is that the complex systems we build can fundamentally +be broken down into agents or services that operate on inputs and outputs. In +most cases, there is actually just one input and output (an RPC client) that +needs handling. + +\subsection{Generated Code} + +When a service is defined, we generate a +\texttt{TProcessor} instance capable of handling RPC requests to that service, +using a few helpers. The fundamental structure (illustrated in pseudo-C++) is +as follows: + +\begin{verbatim} +Service.thrift + => Service.cpp + interface ServiceIf + class ServiceClient : virtual ServiceIf + TProtocol in + TProtocol out + class ServiceProcessor : TProcessor + ServiceIf handler + +ServiceHandler.cpp + class ServiceHandler : virtual ServiceIf + +TServer.cpp + TServer(TProcessor processor, + TServerTransport transport, + TTransportFactory tfactory, + TProtocolFactory pfactory) + serve() +\end{verbatim} + +From the Thrift definition file, we generate the virtual service interface. +A client class is generated, which implements the interface and +uses two \texttt{TProtocol} instances to perform the I/O operations. The +generated processor implements the \texttt{TProcessor} interface. The generated +code has all the logic to handle RPC invocations via the \texttt{process()} +call, and takes as a parameter an instance of the service interface, as +implemented by the application developer. + +The user provides an implementation of the application interface in separate, +non-generated source code. + +\subsection{TServer} + +Finally, the Thrift core libraries provide a \texttt{TServer} abstraction. +The \texttt{TServer} object generally works as follows. + +\begin{itemize} +\item Use the \texttt{TServerTransport} to get a \texttt{TTransport} +\item Use the \texttt{TTransportFactory} to optionally convert the primitive +transport into a suitable application transport (typically the +\texttt{TBufferedTransportFactory} is used here) +\item Use the \texttt{TProtocolFactory} to create an input and output protocol +for the \texttt{TTransport} +\item Invoke the \texttt{process()} method of the \texttt{TProcessor} object +\end{itemize} + +The layers are appropriately separated such that the server code needs to know +nothing about any of the transports, encodings, or applications in play. The +server encapsulates the logic around connection handling, threading, etc. +while the processor deals with RPC. The only code written by the application +developer lives in the definitional Thrift file and the interface +implementation. + +Facebook has deployed multiple \texttt{TServer} implementations, including +the single-threaded \texttt{TSimpleServer}, thread-per-connection +\texttt{TThreadedServer}, and thread-pooling \texttt{TThreadPoolServer}. + +The \texttt{TProcessor} interface is very general by design. There is no +requirement that a \texttt{TServer} take a generated \texttt{TProcessor} +object. Thrift allows the application developer to easily write any type of +server that operates on \texttt{TProtocol} objects (for instance, a server +could simply stream a certain type of object without any actual RPC method +invocation). + +\section{Implementation Details} +\subsection{Target Languages} +Thrift currently supports five target languages: C++, Java, Python, Ruby, and +PHP. At Facebook, we have deployed servers predominantly in C++, Java, and +Python. Thrift services implemented in PHP have also been embedded into the +Apache web server, providing transparent backend access to many of our +frontend constructs using a \texttt{THttpClient} implementation of the +\texttt{TTransport} interface. + +Though Thrift was explicitly designed to be much more efficient and robust +than typical web technologies, as we were designing our XML-based REST web +services API we noticed that Thrift could be easily used to define our +service interface. Though we do not currently employ SOAP envelopes (in the +authors' opinions there is already far too much repetitive enterprise Java +software to do that sort of thing), we were able to quickly extend Thrift to +generate XML Schema Definition files for our service, as well as a framework +for versioning different implementations of our web service. Though public +web services are admittedly tangential to Thrift's core use case and design, +Thrift facilitated rapid iteration and affords us the ability to quickly +migrate our entire XML-based web service onto a higher performance system +should the need arise. + +\subsection{Generated Structs} +We made a conscious decision to make our generated structs as transparent as +possible. All fields are publicly accessible; there are no \texttt{set()} and +\texttt{get()} methods. Similarly, use of the \texttt{isset} object is not +enforced. We do not include any \texttt{FieldNotSetException} construct. +Developers have the option to use these fields to write more robust code, but +the system is robust to the developer ignoring the \texttt{isset} construct +entirely and will provide suitable default behavior in all cases. + +This choice was motivated by the desire to ease application development. Our stated +goal is not to make developers learn a rich new library in their language of +choice, but rather to generate code that allow them to work with the constructs +that are most familiar in each language. + +We also made the \texttt{read()} and \texttt{write()} methods of the generated +objects public so that the objects can be used outside of the context +of RPC clients and servers. Thrift is a useful tool simply for generating +objects that are easily serializable across programming languages. + +\subsection{RPC Method Identification} +Method calls in RPC are implemented by sending the method name as a string. One +issue with this approach is that longer method names require more bandwidth. +We experimented with using fixed-size hashes to identify methods, but in the +end concluded that the savings were not worth the headaches incurred. Reliably +dealing with conflicts across versions of an interface definition file is +impossible without a meta-storage system (i.e. to generate non-conflicting +hashes for the current version of a file, we would have to know about all +conflicts that ever existed in any previous version of the file). + +We wanted to avoid too many unnecessary string comparisons upon +method invocation. To deal with this, we generate maps from strings to function +pointers, so that invocation is effectively accomplished via a constant-time +hash lookup in the common case. This requires the use of a couple interesting +code constructs. Because Java does not have function pointers, process +functions are all private member classes implementing a common interface. + +\begin{verbatim} +private class ping implements ProcessFunction { + public void process(int seqid, + TProtocol iprot, + TProtocol oprot) + throws TException + { ...} +} + +HashMap processMap_ = + new HashMap(); +\end{verbatim} + +In C++, we use a relatively esoteric language construct: member function +pointers. + +\begin{verbatim} +std::map + processMap_; +\end{verbatim} + +Using these techniques, the cost of string processing is minimized, and we +reap the benefit of being able to easily debug corrupt or misunderstood data by +inspecting it for known string method names. + +\subsection{Servers and Multithreading} +Thrift services require basic multithreading to handle simultaneous +requests from multiple clients. For the Python and Java implementations of +Thrift server logic, the standard threading libraries distributed with the +languages provide adequate support. For the C++ implementation, no standard multithread runtime +library exists. Specifically, robust, lightweight, and portable +thread manager and timer class implementations do not exist. We investigated +existing implementations, namely \texttt{boost::thread}, +\texttt{boost::threadpool}, \texttt{ACE\_Thread\_Manager} and +\texttt{ACE\_Timer}. + +While \texttt{boost::threads}\cite{boost.threads} provides clean, +lightweight and robust implementations of multi-thread primitives (mutexes, +conditions, threads) it does not provide a thread manager or timer +implementation. + +\texttt{boost::threadpool}\cite{boost.threadpool} also looked promising but +was not far enough along for our purposes. We wanted to limit the dependency on +third-party libraries as much as possible. Because\\ +\texttt{boost::threadpool} is +not a pure template library and requires runtime libraries and because it is +not yet part of the official Boost distribution we felt it was not ready for +use in Thrift. As \texttt{boost::threadpool} evolves and especially if it is +added to the Boost distribution we may reconsider our decision to not use it. + +ACE has both a thread manager and timer class in addition to multi-thread +primitives. The biggest problem with ACE is that it is ACE. Unlike Boost, ACE +API quality is poor. Everything in ACE has large numbers of dependencies on +everything else in ACE - thus forcing developers to throw out standard +classes, such as STL collections, in favor of ACE's homebrewed implementations. In +addition, unlike Boost, ACE implementations demonstrate little understanding +of the power and pitfalls of C++ programming and take no advantage of modern +templating techniques to ensure compile time safety and reasonable compiler +error messages. For all these reasons, ACE was rejected. Instead, we chose +to implement our own library, described in the following sections. + +\subsection{Thread Primitives} + +The Thrift thread libraries are implemented in the namespace\\ +\texttt{facebook::thrift::concurrency} and have three components: +\begin{itemize} +\item primitives +\item thread pool manager +\item timer manager +\end{itemize} + +As mentioned above, we were hesitant to introduce any additional dependencies +on Thrift. We decided to use \texttt{boost::shared\_ptr} because it is so +useful for multithreaded application, it requires no link-time or +runtime libraries (i.e. it is a pure template library) and it is due +to become part of the C++0x standard. + +We implement standard \texttt{Mutex} and \texttt{Condition} classes, and a + \texttt{Monitor} class. The latter is simply a combination of a mutex and +condition variable and is analogous to the \texttt{Monitor} implementation provided for +the Java \texttt{Object} class. This is also sometimes referred to as a barrier. We +provide a \texttt{Synchronized} guard class to allow Java-like synchronized blocks. +This is just a bit of syntactic sugar, but, like its Java counterpart, clearly +delimits critical sections of code. Unlike its Java counterpart, we still +have the ability to programmatically lock, unlock, block, and signal monitors. + +\begin{verbatim} +void run() { + {Synchronized s(manager->monitor); + if (manager->state == TimerManager::STARTING) { + manager->state = TimerManager::STARTED; + manager->monitor.notifyAll(); + } + } +} +\end{verbatim} + +We again borrowed from Java the distinction between a thread and a runnable +class. A \texttt{Thread} is the actual schedulable object. The +\texttt{Runnable} is the logic to execute within the thread. +The \texttt{Thread} implementation deals with all the platform-specific thread +creation and destruction issues, while the \texttt{Runnable} implementation deals +with the application-specific per-thread logic. The benefit of this approach +is that developers can easily subclass the Runnable class without pulling in +platform-specific super-classes. + +\subsection{Thread, Runnable, and shared\_ptr} +We use \texttt{boost::shared\_ptr} throughout the \texttt{ThreadManager} and +\texttt{TimerManager} implementations to guarantee cleanup of dead objects that can +be accessed by multiple threads. For \texttt{Thread} class implementations, +\texttt{boost::shared\_ptr} usage requires particular attention to make sure +\texttt{Thread} objects are neither leaked nor dereferenced prematurely while +creating and shutting down threads. + +Thread creation requires calling into a C library. (In our case the POSIX +thread library, \texttt{libpthread}, but the same would be true for WIN32 threads). +Typically, the OS makes few, if any, guarantees about when \texttt{ThreadMain}, a C thread's entry-point function, will be called. Therefore, it is +possible that our thread create call, +\texttt{ThreadFactory::newThread()} could return to the caller +well before that time. To ensure that the returned \texttt{Thread} object is not +prematurely cleaned up if the caller gives up its reference prior to the +\texttt{ThreadMain} call, the \texttt{Thread} object makes a weak referenence to +itself in its \texttt{start} method. + +With the weak reference in hand the \texttt{ThreadMain} function can attempt to get +a strong reference before entering the \texttt{Runnable::run} method of the +\texttt{Runnable} object bound to the \texttt{Thread}. If no strong references to the +thread are obtained between exiting \texttt{Thread::start} and entering \texttt{ThreadMain}, the weak reference returns \texttt{null} and the function +exits immediately. + +The need for the \texttt{Thread} to make a weak reference to itself has a +significant impact on the API. Since references are managed through the +\texttt{boost::shared\_ptr} templates, the \texttt{Thread} object must have a reference +to itself wrapped by the same \texttt{boost::shared\_ptr} envelope that is returned +to the caller. This necessitated the use of the factory pattern. +\texttt{ThreadFactory} creates the raw \texttt{Thread} object and a +\texttt{boost::shared\_ptr} wrapper, and calls a private helper method of the class +implementing the \texttt{Thread} interface (in this case, \texttt{PosixThread::weakRef}) + to allow it to make add weak reference to itself through the + \texttt{boost::shared\_ptr} envelope. + +\texttt{Thread} and \texttt{Runnable} objects reference each other. A \texttt{Runnable} +object may need to know about the thread in which it is executing, and a Thread, obviously, +needs to know what \texttt{Runnable} object it is hosting. This interdependency is +further complicated because the lifecycle of each object is independent of the +other. An application may create a set of \texttt{Runnable} object to be reused in different threads, or it may create and forget a \texttt{Runnable} object +once a thread has been created and started for it. + +The \texttt{Thread} class takes a \texttt{boost::shared\_ptr} reference to the hosted +\texttt{Runnable} object in its constructor, while the \texttt{Runnable} class has an +explicit \texttt{thread} method to allow explicit binding of the hosted thread. +\texttt{ThreadFactory::newThread} binds the objects to each other. + +\subsection{ThreadManager} + +\texttt{ThreadManager} creates a pool of worker threads and +allows applications to schedule tasks for execution as free worker threads +become available. The \texttt{ThreadManager} does not implement dynamic +thread pool resizing, but provides primitives so that applications can add +and remove threads based on load. This approach was chosen because +implementing load metrics and thread pool size is very application +specific. For example some applications may want to adjust pool size based +on running-average of work arrival rates that are measured via polled +samples. Others may simply wish to react immediately to work-queue +depth high and low water marks. Rather than trying to create a complex +API abstract enough to capture these different approaches, we +simply leave it up to the particular application and provide the +primitives to enact the desired policy and sample current status. + +\subsection{TimerManager} + +\texttt{TimerManager} allows applications to schedule + \texttt{Runnable} objects for execution at some point in the future. Its specific task +is to allows applications to sample \texttt{ThreadManager} load at regular +intervals and make changes to the thread pool size based on application policy. +Of course, it can be used to generate any number of timer or alarm events. + +The default implementation of \texttt{TimerManager} uses a single thread to +execute expired \texttt{Runnable} objects. Thus, if a timer operation needs to +do a large amount of work and especially if it needs to do blocking I/O, +that should be done in a separate thread. + +\subsection{Nonblocking Operation} +Though the Thrift transport interfaces map more directly to a blocking I/O +model, we have implemented a high performance \texttt{TNonBlockingServer} +in C++ based on \texttt{libevent} and the \texttt{TFramedTransport}. We +implemented this by moving all I/O into one tight event loop using a +state machine. Essentially, the event loop reads framed requests into +\texttt{TMemoryBuffer} objects. Once entire requests are ready, they are +dispatched to the \texttt{TProcessor} object which can read directly from +the data in memory. + +\subsection{Compiler} +The Thrift compiler is implemented in C++ using standard \texttt{lex}/\texttt{yacc} +lexing and parsing. Though it could have been implemented with fewer +lines of code in another language (i.e. Python Lex-Yacc (PLY) or \texttt{ocamlyacc}), using C++ +forces explicit definition of the language constructs. Strongly typing the +parse tree elements (debatably) makes the code more approachable for new +developers. + +Code generation is done using two passes. The first pass looks only for +include files and type definitions. Type definitions are not checked during +this phase, since they may depend upon include files. All included files +are sequentially scanned in a first pass. Once the include tree has been +resolved, a second pass over all files is taken that inserts type definitions +into the parse tree and raises an error on any undefined types. The program is +then generated against the parse tree. + +Due to inherent complexities and potential for circular dependencies, +we explicitly disallow forward declaration. Two Thrift structs cannot +each contain an instance of the other. (Since we do not allow \texttt{null} +struct instances in the generated C++ code, this would actually be impossible.) + +\subsection{TFileTransport} +The \texttt{TFileTransport} logs Thrift requests/structs by +framing incoming data with its length and writing it out to disk. +Using a framed on-disk format allows for better error checking and +helps with the processing of a finite number of discrete events. The\\ +\texttt{TFileWriterTransport} uses a system of swapping in-memory buffers +to ensure good performance while logging large amounts of data. +A Thrift log file is split up into chunks of a specified size; logged messages +are not allowed to cross chunk boundaries. A message that would cross a chunk +boundary will cause padding to be added until the end of the chunk and the +first byte of the message are aligned to the beginning of the next chunk. +Partitioning the file into chunks makes it possible to read and interpret data +from a particular point in the file. + +\section{Facebook Thrift Services} +Thrift has been employed in a large number of applications at Facebook, including +search, logging, mobile, ads and the developer platform. Two specific usages are discussed below. + +\subsection{Search} +Thrift is used as the underlying protocol and transport layer for the Facebook Search service. +The multi-language code generation is well suited for search because it allows for application +development in an efficient server side language (C++) and allows the Facebook PHP-based web application +to make calls to the search service using Thrift PHP libraries. There is also a large +variety of search stats, deployment and testing functionality that is built on top +of generated Python code. Additionally, the Thrift log file format is +used as a redo log for providing real-time search index updates. Thrift has allowed the +search team to leverage each language for its strengths and to develop code at a rapid pace. + +\subsection{Logging} +The Thrift \texttt{TFileTransport} functionality is used for structured logging. Each +service function definition along with its parameters can be considered to be +a structured log entry identified by the function name. This log can then be used for +a variety of purposes, including inline and offline processing, stats aggregation and as a redo log. + +\section{Conclusions} +Thrift has enabled Facebook to build scalable backend +services efficiently by enabling engineers to divide and conquer. Application +developers can focus on application code without worrying about the +sockets layer. We avoid duplicated work by writing buffering and I/O logic +in one place, rather than interspersing it in each application. + +Thrift has been employed in a wide variety of applications at Facebook, +including search, logging, mobile, ads, and the developer platform. We have +found that the marginal performance cost incurred by an extra layer of +software abstraction is far eclipsed by the gains in developer efficiency and +systems reliability. + +\appendix + +\section{Similar Systems} +The following are software systems similar to Thrift. Each is (very!) briefly +described: + +\begin{itemize} +\item \textit{SOAP.} XML-based. Designed for web services via HTTP, excessive +XML parsing overhead. +\item \textit{CORBA.} Relatively comprehensive, debatably overdesigned and +heavyweight. Comparably cumbersome software installation. +\item \textit{COM.} Embraced mainly in Windows client softare. Not an entirely +open solution. +\item \textit{Pillar.} Lightweight and high-performance, but missing versioning +and abstraction. +\item \textit{Protocol Buffers.} Closed-source, owned by Google. Described in +Sawzall paper. +\end{itemize} + +\acks + +Many thanks for feedback on Thrift (and extreme trial by fire) are due to +Martin Smith, Karl Voskuil and Yishan Wong. + +Thrift is a successor to Pillar, a similar system developed +by Adam D'Angelo, first while at Caltech and continued later at Facebook. +Thrift simply would not have happened without Adam's insights. + +\begin{thebibliography}{} + +\bibitem{boost.threads} +Kempf, William, +``Boost.Threads'', +\url{http://www.boost.org/doc/html/threads.html} + +\bibitem{boost.threadpool} +Henkel, Philipp, +``threadpool'', +\url{http://threadpool.sourceforge.net} + +\end{thebibliography} + +\end{document} diff --git a/lib/Makefile.am b/lib/Makefile.am new file mode 100644 index 00000000..3558dd80 --- /dev/null +++ b/lib/Makefile.am @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +SUBDIRS = \ + cpp + +if WITH_MONO +SUBDIRS += csharp +endif + +if WITH_JAVA +SUBDIRS += java +endif + +if WITH_PYTHON +SUBDIRS += py +endif + +if WITH_ERLANG +SUBDIRS += erl +endif + +if WITH_RUBY +SUBDIRS += rb +endif + +if WITH_PERL +SUBDIRS += perl +endif + +# All of the libs that don't use Automake need to go in here +# so they will end up in our release tarballs. +EXTRA_DIST = \ + cocoa \ + hs \ + ocaml \ + php \ + erl \ + st diff --git a/lib/cocoa/README b/lib/cocoa/README new file mode 100644 index 00000000..bbe3c934 --- /dev/null +++ b/lib/cocoa/README @@ -0,0 +1,21 @@ +Thrift Cocoa Software Library + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. diff --git a/lib/cocoa/src/TApplicationException.h b/lib/cocoa/src/TApplicationException.h new file mode 100644 index 00000000..cf1641d9 --- /dev/null +++ b/lib/cocoa/src/TApplicationException.h @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "TException.h" +#import "TProtocol.h" + +enum { + TApplicationException_UNKNOWN = 0, + TApplicationException_UNKNOWN_METHOD = 1, + TApplicationException_INVALID_MESSAGE_TYPE = 2, + TApplicationException_WRONG_METHOD_NAME = 3, + TApplicationException_BAD_SEQUENCE_ID = 4, + TApplicationException_MISSING_RESULT = 5 +}; + +// FIXME +@interface TApplicationException : TException { + int mType; +} + ++ (TApplicationException *) read: (id ) protocol; + +- (void) write: (id ) protocol; + ++ (TApplicationException *) exceptionWithType: (int) type + reason: (NSString *) message; + +@end diff --git a/lib/cocoa/src/TApplicationException.m b/lib/cocoa/src/TApplicationException.m new file mode 100644 index 00000000..70687537 --- /dev/null +++ b/lib/cocoa/src/TApplicationException.m @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "TApplicationException.h" +#import "TProtocolUtil.h" + +@implementation TApplicationException + +- (id) initWithType: (int) type + reason: (NSString *) reason +{ + mType = type; + + NSString * name; + switch (type) { + case TApplicationException_UNKNOWN_METHOD: + name = @"Unknown method"; + break; + case TApplicationException_INVALID_MESSAGE_TYPE: + name = @"Invalid message type"; + break; + case TApplicationException_WRONG_METHOD_NAME: + name = @"Wrong method name"; + break; + case TApplicationException_BAD_SEQUENCE_ID: + name = @"Bad sequence ID"; + break; + case TApplicationException_MISSING_RESULT: + name = @"Missing result"; + break; + default: + name = @"Unknown"; + break; + } + + self = [super initWithName: name reason: reason userInfo: nil]; + return self; +} + + ++ (TApplicationException *) read: (id ) protocol +{ + NSString * reason = nil; + int type = TApplicationException_UNKNOWN; + int fieldType; + int fieldID; + + [protocol readStructBeginReturningName: NULL]; + + while (true) { + [protocol readFieldBeginReturningName: NULL + type: &fieldType + fieldID: &fieldID]; + if (fieldType == TType_STOP) { + break; + } + switch (fieldID) { + case 1: + if (fieldType == TType_STRING) { + reason = [protocol readString]; + } else { + [TProtocolUtil skipType: fieldType onProtocol: protocol]; + } + break; + case 2: + if (fieldType == TType_I32) { + type = [protocol readI32]; + } else { + [TProtocolUtil skipType: fieldType onProtocol: protocol]; + } + break; + default: + [TProtocolUtil skipType: fieldType onProtocol: protocol]; + break; + } + [protocol readFieldEnd]; + } + [protocol readStructEnd]; + + return [TApplicationException exceptionWithType: type reason: reason]; +} + + +- (void) write: (id ) protocol +{ + [protocol writeStructBeginWithName: @"TApplicationException"]; + + if ([self reason] != nil) { + [protocol writeFieldBeginWithName: @"message" + type: TType_STRING + fieldID: 1]; + [protocol writeString: [self reason]]; + [protocol writeFieldEnd]; + } + + [protocol writeFieldBeginWithName: @"type" + type: TType_I32 + fieldID: 2]; + [protocol writeI32: mType]; + [protocol writeFieldEnd]; + + [protocol writeFieldStop]; + [protocol writeStructEnd]; +} + + ++ (TApplicationException *) exceptionWithType: (int) type + reason: (NSString *) reason +{ + return [[[TApplicationException alloc] initWithType: type + reason: reason] autorelease]; +} + +@end diff --git a/lib/cocoa/src/TException.h b/lib/cocoa/src/TException.h new file mode 100644 index 00000000..b069a868 --- /dev/null +++ b/lib/cocoa/src/TException.h @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import + +@interface TException : NSException { +} + ++ (id) exceptionWithName: (NSString *) name; + ++ (id) exceptionWithName: (NSString *) name + reason: (NSString *) reason; + ++ (id) exceptionWithName: (NSString *) name + reason: (NSString *) reason + error: (NSError *) error; + +@end diff --git a/lib/cocoa/src/TException.m b/lib/cocoa/src/TException.m new file mode 100644 index 00000000..7c84199d --- /dev/null +++ b/lib/cocoa/src/TException.m @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "TException.h" + +@implementation TException + ++ (id) exceptionWithName: (NSString *) name +{ + return [self exceptionWithName: name reason: @"unknown" error: nil]; +} + + ++ (id) exceptionWithName: (NSString *) name + reason: (NSString *) reason +{ + return [self exceptionWithName: name reason: reason error: nil]; +} + + ++ (id) exceptionWithName: (NSString *) name + reason: (NSString *) reason + error: (NSError *) error +{ + NSDictionary * userInfo = nil; + if (error != nil) { + userInfo = [NSDictionary dictionaryWithObject: error forKey: @"error"]; + } + + return [super exceptionWithName: name + reason: reason + userInfo: userInfo]; +} + + +- (NSString *) description +{ + NSMutableString * result = [NSMutableString stringWithString: [self name]]; + [result appendFormat: @": %@", [self reason]]; + if ([self userInfo] != nil) { + [result appendFormat: @"\n userInfo = %@", [self userInfo]]; + } + + return result; +} + + +@end diff --git a/lib/cocoa/src/TProcessor.h b/lib/cocoa/src/TProcessor.h new file mode 100644 index 00000000..f8df225e --- /dev/null +++ b/lib/cocoa/src/TProcessor.h @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import + + +@protocol TProcessor + +- (BOOL) processOnInputProtocol: (id ) inProtocol + outputProtocol: (id ) outProtocol; + +@end diff --git a/lib/cocoa/src/protocol/TBinaryProtocol.h b/lib/cocoa/src/protocol/TBinaryProtocol.h new file mode 100644 index 00000000..52cf2669 --- /dev/null +++ b/lib/cocoa/src/protocol/TBinaryProtocol.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "TProtocol.h" +#import "TTransport.h" +#import "TProtocolFactory.h" + + +@interface TBinaryProtocol : NSObject { + id mTransport; + BOOL mStrictRead; + BOOL mStrictWrite; + int32_t mMessageSizeLimit; +} + +- (id) initWithTransport: (id ) transport; + +- (id) initWithTransport: (id ) transport + strictRead: (BOOL) strictRead + strictWrite: (BOOL) strictWrite; + +- (int32_t) messageSizeLimit; +- (void) setMessageSizeLimit: (int32_t) sizeLimit; + +@end; + + +@interface TBinaryProtocolFactory : NSObject { +} + ++ (TBinaryProtocolFactory *) sharedFactory; + +- (TBinaryProtocol *) newProtocolOnTransport: (id ) transport; + +@end diff --git a/lib/cocoa/src/protocol/TBinaryProtocol.m b/lib/cocoa/src/protocol/TBinaryProtocol.m new file mode 100644 index 00000000..ba7f4629 --- /dev/null +++ b/lib/cocoa/src/protocol/TBinaryProtocol.m @@ -0,0 +1,469 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "TBinaryProtocol.h" +#import "TProtocolException.h" + +int32_t VERSION_1 = 0x80010000; +int32_t VERSION_MASK = 0xffff0000; + + +static TBinaryProtocolFactory * gSharedFactory = nil; + +@implementation TBinaryProtocolFactory + ++ (TBinaryProtocolFactory *) sharedFactory { + if (gSharedFactory == nil) { + gSharedFactory = [[TBinaryProtocolFactory alloc] init]; + } + + return gSharedFactory; +} + +- (TBinaryProtocol *) newProtocolOnTransport: (id ) transport { + return [[[TBinaryProtocol alloc] initWithTransport: transport] autorelease]; +} + +@end + + + +@implementation TBinaryProtocol + +- (id) initWithTransport: (id ) transport +{ + return [self initWithTransport: transport strictRead: NO strictWrite: NO]; +} + +- (id) initWithTransport: (id ) transport + strictRead: (BOOL) strictRead + strictWrite: (BOOL) strictWrite +{ + self = [super init]; + mTransport = [transport retain]; + mStrictRead = strictRead; + mStrictWrite = strictWrite; + return self; +} + + +- (int32_t) messageSizeLimit +{ + return mMessageSizeLimit; +} + + +- (void) setMessageSizeLimit: (int32_t) sizeLimit +{ + mMessageSizeLimit = sizeLimit; +} + + +- (void) dealloc +{ + [mTransport release]; + [super dealloc]; +} + + +- (id ) transport +{ + return mTransport; +} + + +- (NSString *) readStringBody: (int) size +{ + char buff[size+1]; + [mTransport readAll: (uint8_t *) buff offset: 0 length: size]; + buff[size] = 0; + return [NSString stringWithUTF8String: buff]; +} + + +- (void) readMessageBeginReturningName: (NSString **) name + type: (int *) type + sequenceID: (int *) sequenceID +{ + int32_t size = [self readI32]; + if (size < 0) { + int version = size & VERSION_MASK; + if (version != VERSION_1) { + @throw [TProtocolException exceptionWithName: @"TProtocolException" + reason: @"Bad version in readMessageBegin"]; + } + if (type != NULL) { + *type = version & 0x00FF; + } + NSString * messageName = [self readString]; + if (name != NULL) { + *name = messageName; + } + int seqID = [self readI32]; + if (sequenceID != NULL) { + *sequenceID = seqID; + } + } else { + if (mStrictRead) { + @throw [TProtocolException exceptionWithName: @"TProtocolException" + reason: @"Missing version in readMessageBegin, old client?"]; + } + if ([self messageSizeLimit] > 0 && size > [self messageSizeLimit]) { + @throw [TProtocolException exceptionWithName: @"TProtocolException" + reason: [NSString stringWithFormat: @"Message too big. Size limit is: %d Message size is: %d", + mMessageSizeLimit, + size]]; + } + NSString * messageName = [self readStringBody: size]; + if (name != NULL) { + *name = messageName; + } + int messageType = [self readByte]; + if (type != NULL) { + *type = messageType; + } + int seqID = [self readI32]; + if (sequenceID != NULL) { + *sequenceID = seqID; + } + } +} + + +- (void) readMessageEnd {} + + +- (void) readStructBeginReturningName: (NSString **) name +{ + if (name != NULL) { + *name = nil; + } +} + + +- (void) readStructEnd {} + + +- (void) readFieldBeginReturningName: (NSString **) name + type: (int *) fieldType + fieldID: (int *) fieldID +{ + if (name != NULL) { + *name = nil; + } + int ft = [self readByte]; + if (fieldType != NULL) { + *fieldType = ft; + } + if (ft != TType_STOP) { + int fid = [self readI16]; + if (fieldID != NULL) { + *fieldID = fid; + } + } +} + + +- (void) readFieldEnd {} + + +- (int32_t) readI32 +{ + uint8_t i32rd[4]; + [mTransport readAll: i32rd offset: 0 length: 4]; + return + ((i32rd[0] & 0xff) << 24) | + ((i32rd[1] & 0xff) << 16) | + ((i32rd[2] & 0xff) << 8) | + ((i32rd[3] & 0xff)); +} + + +- (NSString *) readString +{ + int size = [self readI32]; + return [self readStringBody: size]; +} + + +- (BOOL) readBool +{ + return [self readByte] == 1; +} + +- (uint8_t) readByte +{ + uint8_t myByte; + [mTransport readAll: &myByte offset: 0 length: 1]; + return myByte; +} + +- (short) readI16 +{ + uint8_t buff[2]; + [mTransport readAll: buff offset: 0 length: 2]; + return (short) + (((buff[0] & 0xff) << 8) | + ((buff[1] & 0xff))); + return 0; +} + +- (int64_t) readI64; +{ + uint8_t i64rd[8]; + [mTransport readAll: i64rd offset: 0 length: 8]; + return + ((int64_t)(i64rd[0] & 0xff) << 56) | + ((int64_t)(i64rd[1] & 0xff) << 48) | + ((int64_t)(i64rd[2] & 0xff) << 40) | + ((int64_t)(i64rd[3] & 0xff) << 32) | + ((int64_t)(i64rd[4] & 0xff) << 24) | + ((int64_t)(i64rd[5] & 0xff) << 16) | + ((int64_t)(i64rd[6] & 0xff) << 8) | + ((int64_t)(i64rd[7] & 0xff)); +} + +- (double) readDouble; +{ + // FIXME - will this get us into trouble on PowerPC? + int64_t ieee754 = [self readI64]; + return *((double *) &ieee754); +} + + +- (NSData *) readBinary +{ + int32_t size = [self readI32]; + uint8_t * buff = malloc(size); + if (buff == NULL) { + @throw [TProtocolException + exceptionWithName: @"TProtocolException" + reason: [NSString stringWithFormat: @"Out of memory. Unable to allocate %d bytes trying to read binary data.", + size]]; + } + [mTransport readAll: buff offset: 0 length: size]; + return [NSData dataWithBytesNoCopy: buff length: size]; +} + + +- (void) readMapBeginReturningKeyType: (int *) keyType + valueType: (int *) valueType + size: (int *) size +{ + int kt = [self readByte]; + int vt = [self readByte]; + int s = [self readI32]; + if (keyType != NULL) { + *keyType = kt; + } + if (valueType != NULL) { + *valueType = vt; + } + if (size != NULL) { + *size = s; + } +} + +- (void) readMapEnd {} + + +- (void) readSetBeginReturningElementType: (int *) elementType + size: (int *) size +{ + int et = [self readByte]; + int s = [self readI32]; + if (elementType != NULL) { + *elementType = et; + } + if (size != NULL) { + *size = s; + } +} + + +- (void) readSetEnd {} + + +- (void) readListBeginReturningElementType: (int *) elementType + size: (int *) size +{ + int et = [self readByte]; + int s = [self readI32]; + if (elementType != NULL) { + *elementType = et; + } + if (size != NULL) { + *size = s; + } +} + + +- (void) readListEnd {} + + +- (void) writeByte: (uint8_t) value +{ + [mTransport write: &value offset: 0 length: 1]; +} + + +- (void) writeMessageBeginWithName: (NSString *) name + type: (int) messageType + sequenceID: (int) sequenceID +{ + if (mStrictWrite) { + int version = VERSION_1 | messageType; + [self writeI32: version]; + [self writeString: name]; + [self writeI32: sequenceID]; + } else { + [self writeString: name]; + [self writeByte: messageType]; + [self writeI32: sequenceID]; + } +} + + +- (void) writeMessageEnd {} + + +- (void) writeStructBeginWithName: (NSString *) name {} + + +- (void) writeStructEnd {} + + +- (void) writeFieldBeginWithName: (NSString *) name + type: (int) fieldType + fieldID: (int) fieldID +{ + [self writeByte: fieldType]; + [self writeI16: fieldID]; +} + + +- (void) writeI32: (int32_t) value +{ + uint8_t buff[4]; + buff[0] = 0xFF & (value >> 24); + buff[1] = 0xFF & (value >> 16); + buff[2] = 0xFF & (value >> 8); + buff[3] = 0xFF & value; + [mTransport write: buff offset: 0 length: 4]; +} + +- (void) writeI16: (short) value +{ + uint8_t buff[2]; + buff[0] = 0xff & (value >> 8); + buff[1] = 0xff & value; + [mTransport write: buff offset: 0 length: 2]; +} + + +- (void) writeI64: (int64_t) value +{ + uint8_t buff[8]; + buff[0] = 0xFF & (value >> 56); + buff[1] = 0xFF & (value >> 48); + buff[2] = 0xFF & (value >> 40); + buff[3] = 0xFF & (value >> 32); + buff[4] = 0xFF & (value >> 24); + buff[5] = 0xFF & (value >> 16); + buff[6] = 0xFF & (value >> 8); + buff[7] = 0xFF & value; + [mTransport write: buff offset: 0 length: 8]; +} + +- (void) writeDouble: (double) value +{ + // spit out IEEE 754 bits - FIXME - will this get us in trouble on + // PowerPC? + [self writeI64: *((int64_t *) &value)]; +} + + +- (void) writeString: (NSString *) value +{ + if (value != nil) { + const char * utf8Bytes = [value UTF8String]; + size_t length = strlen(utf8Bytes); + [self writeI32: length]; + [mTransport write: (uint8_t *) utf8Bytes offset: 0 length: length]; + } else { + // instead of crashing when we get null, let's write out a zero + // length string + [self writeI32: 0]; + } +} + + +- (void) writeBinary: (NSData *) data +{ + [self writeI32: [data length]]; + [mTransport write: [data bytes] offset: 0 length: [data length]]; +} + +- (void) writeFieldStop +{ + [self writeByte: TType_STOP]; +} + + +- (void) writeFieldEnd {} + + +- (void) writeMapBeginWithKeyType: (int) keyType + valueType: (int) valueType + size: (int) size +{ + [self writeByte: keyType]; + [self writeByte: valueType]; + [self writeI32: size]; +} + +- (void) writeMapEnd {} + + +- (void) writeSetBeginWithElementType: (int) elementType + size: (int) size +{ + [self writeByte: elementType]; + [self writeI32: size]; +} + +- (void) writeSetEnd {} + + +- (void) writeListBeginWithElementType: (int) elementType + size: (int) size +{ + [self writeByte: elementType]; + [self writeI32: size]; +} + +- (void) writeListEnd {} + + +- (void) writeBool: (BOOL) value +{ + [self writeByte: (value ? 1 : 0)]; +} + +@end diff --git a/lib/cocoa/src/protocol/TProtocol.h b/lib/cocoa/src/protocol/TProtocol.h new file mode 100644 index 00000000..cc8cdb4b --- /dev/null +++ b/lib/cocoa/src/protocol/TProtocol.h @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import + +#import "TTransport.h" + + +enum { + TMessageType_CALL = 1, + TMessageType_REPLY = 2, + TMessageType_EXCEPTION = 3, + TMessageType_ONEWAY = 4 +}; + +enum { + TType_STOP = 0, + TType_VOID = 1, + TType_BOOL = 2, + TType_BYTE = 3, + TType_DOUBLE = 4, + TType_I16 = 6, + TType_I32 = 8, + TType_I64 = 10, + TType_STRING = 11, + TType_STRUCT = 12, + TType_MAP = 13, + TType_SET = 14, + TType_LIST = 15 +}; + + +@protocol TProtocol + +- (id ) transport; + +- (void) readMessageBeginReturningName: (NSString **) name + type: (int *) type + sequenceID: (int *) sequenceID; +- (void) readMessageEnd; + +- (void) readStructBeginReturningName: (NSString **) name; +- (void) readStructEnd; + +- (void) readFieldBeginReturningName: (NSString **) name + type: (int *) fieldType + fieldID: (int *) fieldID; +- (void) readFieldEnd; + +- (NSString *) readString; + +- (BOOL) readBool; + +- (unsigned char) readByte; + +- (short) readI16; + +- (int32_t) readI32; + +- (int64_t) readI64; + +- (double) readDouble; + +- (NSData *) readBinary; + +- (void) readMapBeginReturningKeyType: (int *) keyType + valueType: (int *) valueType + size: (int *) size; +- (void) readMapEnd; + + +- (void) readSetBeginReturningElementType: (int *) elementType + size: (int *) size; +- (void) readSetEnd; + + +- (void) readListBeginReturningElementType: (int *) elementType + size: (int *) size; +- (void) readListEnd; + + +- (void) writeMessageBeginWithName: (NSString *) name + type: (int) messageType + sequenceID: (int) sequenceID; +- (void) writeMessageEnd; + +- (void) writeStructBeginWithName: (NSString *) name; +- (void) writeStructEnd; + +- (void) writeFieldBeginWithName: (NSString *) name + type: (int) fieldType + fieldID: (int) fieldID; + +- (void) writeI32: (int32_t) value; + +- (void) writeI64: (int64_t) value; + +- (void) writeI16: (short) value; + +- (void) writeByte: (uint8_t) value; + +- (void) writeString: (NSString *) value; + +- (void) writeDouble: (double) value; + +- (void) writeBool: (BOOL) value; + +- (void) writeBinary: (NSData *) data; + +- (void) writeFieldStop; + +- (void) writeFieldEnd; + +- (void) writeMapBeginWithKeyType: (int) keyType + valueType: (int) valueType + size: (int) size; +- (void) writeMapEnd; + + +- (void) writeSetBeginWithElementType: (int) elementType + size: (int) size; +- (void) writeSetEnd; + + +- (void) writeListBeginWithElementType: (int) elementType + size: (int) size; + +- (void) writeListEnd; + + +@end + diff --git a/lib/cocoa/src/protocol/TProtocolException.h b/lib/cocoa/src/protocol/TProtocolException.h new file mode 100644 index 00000000..ad354fc2 --- /dev/null +++ b/lib/cocoa/src/protocol/TProtocolException.h @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "TException.h" + +@interface TProtocolException : TException { +} + +@end diff --git a/lib/cocoa/src/protocol/TProtocolException.m b/lib/cocoa/src/protocol/TProtocolException.m new file mode 100644 index 00000000..681487a4 --- /dev/null +++ b/lib/cocoa/src/protocol/TProtocolException.m @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "TProtocolException.h" + +@implementation TProtocolException +@end diff --git a/lib/cocoa/src/protocol/TProtocolFactory.h b/lib/cocoa/src/protocol/TProtocolFactory.h new file mode 100644 index 00000000..2d125e96 --- /dev/null +++ b/lib/cocoa/src/protocol/TProtocolFactory.h @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import +#import "TProtocol.h" +#import "TTransport.h" + + +@protocol TProtocolFactory + +- (id ) newProtocolOnTransport: (id ) transport; + +@end diff --git a/lib/cocoa/src/protocol/TProtocolUtil.h b/lib/cocoa/src/protocol/TProtocolUtil.h new file mode 100644 index 00000000..c2d2521c --- /dev/null +++ b/lib/cocoa/src/protocol/TProtocolUtil.h @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "TProtocol.h" +#import "TTransport.h" + +@interface TProtocolUtil : NSObject { + +} + ++ (void) skipType: (int) type onProtocol: (id ) protocol; + +@end; diff --git a/lib/cocoa/src/protocol/TProtocolUtil.m b/lib/cocoa/src/protocol/TProtocolUtil.m new file mode 100644 index 00000000..13d70954 --- /dev/null +++ b/lib/cocoa/src/protocol/TProtocolUtil.m @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "TProtocolUtil.h" + +@implementation TProtocolUtil + ++ (void) skipType: (int) type onProtocol: (id ) protocol +{ + switch (type) { + case TType_BOOL: + [protocol readBool]; + break; + case TType_BYTE: + [protocol readByte]; + break; + case TType_I16: + [protocol readI16]; + break; + case TType_I32: + [protocol readI32]; + break; + case TType_I64: + [protocol readI64]; + break; + case TType_DOUBLE: + [protocol readDouble]; + break; + case TType_STRING: + [protocol readString]; + break; + case TType_STRUCT: + [protocol readStructBeginReturningName: NULL]; + while (true) { + int fieldType; + [protocol readFieldBeginReturningName: nil type: &fieldType fieldID: nil]; + if (fieldType == TType_STOP) { + break; + } + [TProtocolUtil skipType: fieldType onProtocol: protocol]; + [protocol readFieldEnd]; + } + [protocol readStructEnd]; + break; + case TType_MAP: + { + int keyType; + int valueType; + int size; + [protocol readMapBeginReturningKeyType: &keyType valueType: &valueType size: &size]; + int i; + for (i = 0; i < size; i++) { + [TProtocolUtil skipType: keyType onProtocol: protocol]; + [TProtocolUtil skipType: valueType onProtocol: protocol]; + } + [protocol readMapEnd]; + } + break; + case TType_SET: + { + int elemType; + int size; + [protocol readSetBeginReturningElementType: &elemType size: &size]; + int i; + for (i = 0; i < size; i++) { + [TProtocolUtil skipType: elemType onProtocol: protocol]; + } + [protocol readSetEnd]; + } + break; + case TType_LIST: + { + int elemType; + int size; + [protocol readListBeginReturningElementType: &elemType size: &size]; + int i; + for (i = 0; i < size; i++) { + [TProtocolUtil skipType: elemType onProtocol: protocol]; + } + [protocol readListEnd]; + } + break; + default: + return; + } +} + +@end diff --git a/lib/cocoa/src/server/TSocketServer.h b/lib/cocoa/src/server/TSocketServer.h new file mode 100644 index 00000000..3d4a9e0e --- /dev/null +++ b/lib/cocoa/src/server/TSocketServer.h @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import +#import "TProtocolFactory.h" +#import "TProcessor.h" + + +@interface TSocketServer : NSObject { + NSSocketPort * mServerSocket; + NSFileHandle * mSocketFileHandle; + id mInputProtocolFactory; + id mOutputProtocolFactory; + id mProcessor; +} + +- (id) initWithPort: (int) port + protocolFactory: (id ) protocolFactory + processor: (id ) processor; + +@end + + + diff --git a/lib/cocoa/src/server/TSocketServer.m b/lib/cocoa/src/server/TSocketServer.m new file mode 100644 index 00000000..97d8bae7 --- /dev/null +++ b/lib/cocoa/src/server/TSocketServer.m @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import +#import "TSocketServer.h" +#import "TNSFileHandleTransport.h" +#import "TProtocol.h" +#import "TTransportException.h" + + +@implementation TSocketServer + +- (id) initWithPort: (int) port + protocolFactory: (id ) protocolFactory + processor: (id ) processor; +{ + self = [super init]; + + mInputProtocolFactory = [protocolFactory retain]; + mOutputProtocolFactory = [protocolFactory retain]; + mProcessor = [processor retain]; + + // create a socket + mServerSocket = [[NSSocketPort alloc] initWithTCPPort: port]; + // FIXME - move this separate start method and add method to close + // and cleanup any open ports + + if (mServerSocket == nil) { + NSLog(@"Unable to listen on TCP port %d", port); + } else { + NSLog(@"Listening on TCP port %d", port); + + // wrap it in a file handle so we can get messages from it + mSocketFileHandle = [[NSFileHandle alloc] initWithFileDescriptor: [mServerSocket socket] + closeOnDealloc: YES]; + + // register for notifications of accepted incoming connections + [[NSNotificationCenter defaultCenter] addObserver: self + selector: @selector(connectionAccepted:) + name: NSFileHandleConnectionAcceptedNotification + object: mSocketFileHandle]; + + // tell socket to listen + [mSocketFileHandle acceptConnectionInBackgroundAndNotify]; + } + + return self; +} + + +- (void) dealloc { + [mInputProtocolFactory release]; + [mOutputProtocolFactory release]; + [mProcessor release]; + [mSocketFileHandle release]; + [mServerSocket release]; + [super dealloc]; +} + + +- (void) connectionAccepted: (NSNotification *) aNotification +{ + NSFileHandle * socket = [[aNotification userInfo] objectForKey: NSFileHandleNotificationFileHandleItem]; + + // now that we have a client connected, spin off a thread to handle activity + [NSThread detachNewThreadSelector: @selector(handleClientConnection:) + toTarget: self + withObject: socket]; + + [[aNotification object] acceptConnectionInBackgroundAndNotify]; +} + + +- (void) handleClientConnection: (NSFileHandle *) clientSocket +{ + NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init]; + + TNSFileHandleTransport * transport = [[TNSFileHandleTransport alloc] initWithFileHandle: clientSocket]; + + id inProtocol = [mInputProtocolFactory newProtocolOnTransport: transport]; + id outProtocol = [mOutputProtocolFactory newProtocolOnTransport: transport]; + + @try { + while ([mProcessor processOnInputProtocol: inProtocol outputProtocol: outProtocol]); + } + @catch (TTransportException * te) { + NSLog(@"%@", te); + } + + [pool release]; +} + + + +@end + + + diff --git a/lib/cocoa/src/transport/THTTPClient.h b/lib/cocoa/src/transport/THTTPClient.h new file mode 100644 index 00000000..86f3f054 --- /dev/null +++ b/lib/cocoa/src/transport/THTTPClient.h @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import +#import "TTransport.h" + +@interface THTTPClient : NSObject { + NSURL * mURL; + NSMutableURLRequest * mRequest; + NSMutableData * mRequestData; + NSData * mResponseData; + int mResponseDataOffset; + NSString * mUserAgent; + int mTimeout; +} + +- (id) initWithURL: (NSURL *) aURL; + +- (id) initWithURL: (NSURL *) aURL + userAgent: (NSString *) userAgent + timeout: (int) timeout; + +- (void) setURL: (NSURL *) aURL; + +@end + diff --git a/lib/cocoa/src/transport/THTTPClient.m b/lib/cocoa/src/transport/THTTPClient.m new file mode 100644 index 00000000..6391bead --- /dev/null +++ b/lib/cocoa/src/transport/THTTPClient.m @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "THTTPClient.h" +#import "TTransportException.h" + +@implementation THTTPClient + + +- (void) setupRequest +{ + if (mRequest != nil) { + [mRequest release]; + } + + // set up our request object that we'll use for each request + mRequest = [[NSMutableURLRequest alloc] initWithURL: mURL]; + [mRequest setHTTPMethod: @"POST"]; + [mRequest setValue: @"application/x-thrift" forHTTPHeaderField: @"Content-Type"]; + [mRequest setValue: @"application/x-thrift" forHTTPHeaderField: @"Accept"]; + + NSString * userAgent = mUserAgent; + if (!userAgent) { + userAgent = @"Cocoa/THTTPClient"; + } + [mRequest setValue: userAgent forHTTPHeaderField: @"User-Agent"]; + + [mRequest setCachePolicy: NSURLRequestReloadIgnoringCacheData]; + if (mTimeout) { + [mRequest setTimeoutInterval: mTimeout]; + } +} + + +- (id) initWithURL: (NSURL *) aURL +{ + return [self initWithURL: aURL + userAgent: nil + timeout: 0]; +} + + +- (id) initWithURL: (NSURL *) aURL + userAgent: (NSString *) userAgent + timeout: (int) timeout +{ + self = [super init]; + if (!self) { + return nil; + } + + mTimeout = timeout; + if (userAgent) { + mUserAgent = [userAgent retain]; + } + mURL = [aURL retain]; + + [self setupRequest]; + + // create our request data buffer + mRequestData = [[NSMutableData alloc] initWithCapacity: 1024]; + + return self; +} + + +- (void) setURL: (NSURL *) aURL +{ + [aURL retain]; + [mURL release]; + mURL = aURL; + + [self setupRequest]; +} + + +- (void) dealloc +{ + [mURL release]; + [mUserAgent release]; + [mRequest release]; + [mRequestData release]; + [mResponseData release]; + [super dealloc]; +} + + +- (int) readAll: (uint8_t *) buf offset: (int) off length: (int) len +{ + NSRange r; + r.location = mResponseDataOffset; + r.length = len; + + [mResponseData getBytes: buf+off range: r]; + mResponseDataOffset += len; + + return len; +} + + +- (void) write: (const uint8_t *) data offset: (unsigned int) offset length: (unsigned int) length +{ + [mRequestData appendBytes: data+offset length: length]; +} + + +- (void) flush +{ + [mRequest setHTTPBody: mRequestData]; // not sure if it copies the data + + // make the HTTP request + NSURLResponse * response; + NSError * error; + NSData * responseData = + [NSURLConnection sendSynchronousRequest: mRequest returningResponse: &response error: &error]; + + [mRequestData setLength: 0]; + + if (responseData == nil) { + @throw [TTransportException exceptionWithName: @"TTransportException" + reason: @"Could not make HTTP request" + error: error]; + } + if (![response isKindOfClass: [NSHTTPURLResponse class]]) { + @throw [TTransportException exceptionWithName: @"TTransportException" + reason: @"Unexpected NSURLResponse type"]; + } + + NSHTTPURLResponse * httpResponse = (NSHTTPURLResponse *) response; + if ([httpResponse statusCode] != 200) { + @throw [TTransportException exceptionWithName: @"TTransportException" + reason: [NSString stringWithFormat: @"Bad response from HTTP server: %d", + [httpResponse statusCode]]]; + } + + // phew! + [mResponseData release]; + mResponseData = [responseData retain]; + mResponseDataOffset = 0; +} + + +@end diff --git a/lib/cocoa/src/transport/TNSFileHandleTransport.h b/lib/cocoa/src/transport/TNSFileHandleTransport.h new file mode 100644 index 00000000..64a6af3c --- /dev/null +++ b/lib/cocoa/src/transport/TNSFileHandleTransport.h @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +#import +#import "TTransport.h" + +@interface TNSFileHandleTransport : NSObject { + NSFileHandle * mInputFileHandle; + NSFileHandle * mOutputFileHandle; +} + +- (id) initWithFileHandle: (NSFileHandle *) fileHandle; + +- (id) initWithInputFileHandle: (NSFileHandle *) inputFileHandle + outputFileHandle: (NSFileHandle *) outputFileHandle; + + +@end diff --git a/lib/cocoa/src/transport/TNSFileHandleTransport.m b/lib/cocoa/src/transport/TNSFileHandleTransport.m new file mode 100644 index 00000000..15339341 --- /dev/null +++ b/lib/cocoa/src/transport/TNSFileHandleTransport.m @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +#import "TNSFileHandleTransport.h" +#import "TTransportException.h" + + +@implementation TNSFileHandleTransport + +- (id) initWithFileHandle: (NSFileHandle *) fileHandle +{ + return [self initWithInputFileHandle: fileHandle + outputFileHandle: fileHandle]; +} + + +- (id) initWithInputFileHandle: (NSFileHandle *) inputFileHandle + outputFileHandle: (NSFileHandle *) outputFileHandle +{ + self = [super init]; + + mInputFileHandle = [inputFileHandle retain]; + mOutputFileHandle = [outputFileHandle retain]; + + return self; +} + + +- (void) dealloc { + [mInputFileHandle release]; + [mOutputFileHandle release]; + [super dealloc]; +} + + +- (int) readAll: (uint8_t *) buf offset: (int) off length: (int) len +{ + int got = 0; + while (got < len) { + NSData * d = [mInputFileHandle readDataOfLength: len-got]; + if ([d length] == 0) { + @throw [TTransportException exceptionWithName: @"TTransportException" + reason: @"Cannot read. No more data."]; + } + [d getBytes: buf+got]; + got += [d length]; + } + return got; +} + + +- (void) write: (uint8_t *) data offset: (unsigned int) offset length: (unsigned int) length +{ + NSData * dataObject = [[NSData alloc] initWithBytesNoCopy: data+offset + length: length + freeWhenDone: NO]; + + [mOutputFileHandle writeData: dataObject]; + + + [dataObject release]; +} + + +- (void) flush +{ + +} + +@end diff --git a/lib/cocoa/src/transport/TNSStreamTransport.h b/lib/cocoa/src/transport/TNSStreamTransport.h new file mode 100644 index 00000000..295a185c --- /dev/null +++ b/lib/cocoa/src/transport/TNSStreamTransport.h @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import +#import "TTransport.h" + +@interface TNSStreamTransport : NSObject { + NSInputStream * mInput; + NSOutputStream * mOutput; +} + +- (id) initWithInputStream: (NSInputStream *) input + outputStream: (NSOutputStream *) output; + +- (id) initWithInputStream: (NSInputStream *) input; + +- (id) initWithOutputStream: (NSOutputStream *) output; + +@end + + + diff --git a/lib/cocoa/src/transport/TNSStreamTransport.m b/lib/cocoa/src/transport/TNSStreamTransport.m new file mode 100644 index 00000000..52a02e27 --- /dev/null +++ b/lib/cocoa/src/transport/TNSStreamTransport.m @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "TNSStreamTransport.h" +#import "TTransportException.h" + + +@implementation TNSStreamTransport + +- (id) initWithInputStream: (NSInputStream *) input + outputStream: (NSOutputStream *) output +{ + [super init]; + mInput = [input retain]; + mOutput = [output retain]; + return self; +} + +- (id) initWithInputStream: (NSInputStream *) input +{ + return [self initWithInputStream: input outputStream: nil]; +} + +- (id) initWithOutputStream: (NSOutputStream *) output +{ + return [self initWithInputStream: nil outputStream: output]; +} + +- (void) dealloc +{ + [mInput release]; + [mOutput release]; + [super dealloc]; +} + + +- (int) readAll: (uint8_t *) buf offset: (int) off length: (int) len +{ + int got = 0; + int ret = 0; + while (got < len) { + ret = [mInput read: buf+off+got maxLength: len-got]; + if (ret <= 0) { + @throw [TTransportException exceptionWithReason: @"Cannot read. Remote side has closed."]; + } + got += ret; + } + return got; +} + + +// FIXME:geech:20071019 - make this write all +- (void) write: (uint8_t *) data offset: (unsigned int) offset length: (unsigned int) length +{ + int result = [mOutput write: data+offset maxLength: length]; + if (result == -1) { + @throw [TTransportException exceptionWithReason: @"Error writing to transport output stream." + error: [mOutput streamError]]; + } else if (result == 0) { + @throw [TTransportException exceptionWithReason: @"End of output stream."]; + } else if (result != length) { + @throw [TTransportException exceptionWithReason: @"Output stream did not write all of our data."]; + } +} + +- (void) flush +{ + // no flush for you! +} + +@end diff --git a/lib/cocoa/src/transport/TSocketClient.h b/lib/cocoa/src/transport/TSocketClient.h new file mode 100644 index 00000000..a883acbb --- /dev/null +++ b/lib/cocoa/src/transport/TSocketClient.h @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import +#import "TNSStreamTransport.h" + +@interface TSocketClient : TNSStreamTransport { +} + +- (id) initWithHostname: (NSString *) hostname + port: (int) port; + +@end + + + diff --git a/lib/cocoa/src/transport/TSocketClient.m b/lib/cocoa/src/transport/TSocketClient.m new file mode 100644 index 00000000..7c07c561 --- /dev/null +++ b/lib/cocoa/src/transport/TSocketClient.m @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import +#import "TSocketClient.h" + +@implementation TSocketClient + +- (id) initWithHostname: (NSString *) hostname + port: (int) port +{ + NSInputStream * input = nil; + NSOutputStream * output = nil; + + [NSStream getStreamsToHost: [NSHost hostWithName: hostname] + port: port + inputStream: &input + outputStream: &output]; + + self = [super initWithInputStream: input outputStream: output]; + [input open]; + [output open]; + + return self; +} + + +@end + + + diff --git a/lib/cocoa/src/transport/TTransport.h b/lib/cocoa/src/transport/TTransport.h new file mode 100644 index 00000000..61ebbd21 --- /dev/null +++ b/lib/cocoa/src/transport/TTransport.h @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@protocol TTransport + + /** + * Guarantees that all of len bytes are read + * + * @param buf Buffer to read into + * @param off Index in buffer to start storing bytes at + * @param len Maximum number of bytes to read + * @return The number of bytes actually read, which must be equal to len + * @throws TTransportException if there was an error reading data + */ +- (int) readAll: (uint8_t *) buf offset: (int) off length: (int) len; + +- (void) write: (const uint8_t *) data offset: (unsigned int) offset length: (unsigned int) length; + +- (void) flush; +@end diff --git a/lib/cocoa/src/transport/TTransportException.h b/lib/cocoa/src/transport/TTransportException.h new file mode 100644 index 00000000..6749fe28 --- /dev/null +++ b/lib/cocoa/src/transport/TTransportException.h @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "TException.h" + +@interface TTransportException : TException { +} + ++ (id) exceptionWithReason: (NSString *) reason + error: (NSError *) error; + ++ (id) exceptionWithReason: (NSString *) reason; + +@end diff --git a/lib/cocoa/src/transport/TTransportException.m b/lib/cocoa/src/transport/TTransportException.m new file mode 100644 index 00000000..aa67149e --- /dev/null +++ b/lib/cocoa/src/transport/TTransportException.m @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#import "TTransportException.h" + +@implementation TTransportException + ++ (id) exceptionWithReason: (NSString *) reason + error: (NSError *) error +{ + NSDictionary * userInfo = nil; + if (error != nil) { + userInfo = [NSDictionary dictionaryWithObject: error forKey: @"error"]; + } + + return [super exceptionWithName: @"TTransportException" + reason: reason + userInfo: userInfo]; +} + + ++ (id) exceptionWithReason: (NSString *) reason +{ + return [self exceptionWithReason: reason error: nil]; +} + +@end diff --git a/lib/cpp/Makefile.am b/lib/cpp/Makefile.am new file mode 100644 index 00000000..dc0b6ae5 --- /dev/null +++ b/lib/cpp/Makefile.am @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +ACLOCAL_AMFLAGS = -I ./aclocal + +pkgconfigdir = $(libdir)/pkgconfig + +lib_LTLIBRARIES = libthrift.la +pkgconfig_DATA = thrift.pc + +## We only build the extra libraries if we have the dependencies, +## but we install all of the headers unconditionally. +if AMX_HAVE_LIBEVENT +lib_LTLIBRARIES += libthriftnb.la +pkgconfig_DATA += thrift-nb.pc +endif +if AMX_HAVE_ZLIB +lib_LTLIBRARIES += libthriftz.la +pkgconfig_DATA += thrift-z.pc +endif + +AM_CXXFLAGS = -Wall +AM_CPPFLAGS = $(BOOST_CPPFLAGS) -I$(srcdir)/src + +# Define the source files for the module + +libthrift_la_SOURCES = src/Thrift.cpp \ + src/concurrency/Mutex.cpp \ + src/concurrency/Monitor.cpp \ + src/concurrency/PosixThreadFactory.cpp \ + src/concurrency/ThreadManager.cpp \ + src/concurrency/TimerManager.cpp \ + src/concurrency/Util.cpp \ + src/protocol/TBinaryProtocol.cpp \ + src/protocol/TCompactProtocol.cpp \ + src/protocol/TDebugProtocol.cpp \ + src/protocol/TDenseProtocol.cpp \ + src/protocol/TJSONProtocol.cpp \ + src/protocol/TBase64Utils.cpp \ + src/transport/TTransportException.cpp \ + src/transport/TFDTransport.cpp \ + src/transport/TFileTransport.cpp \ + src/transport/TSimpleFileTransport.cpp \ + src/transport/THttpClient.cpp \ + src/transport/TSocket.cpp \ + src/transport/TSocketPool.cpp \ + src/transport/TServerSocket.cpp \ + src/transport/TTransportUtils.cpp \ + src/transport/TBufferTransports.cpp \ + src/server/TServer.cpp \ + src/server/TSimpleServer.cpp \ + src/server/TThreadPoolServer.cpp \ + src/server/TThreadedServer.cpp \ + src/processor/PeekProcessor.cpp + +libthriftnb_la_SOURCES = src/server/TNonblockingServer.cpp + +libthriftz_la_SOURCES = src/transport/TZlibTransport.cpp + + +# Flags for the various libraries +libthriftnb_la_CPPFLAGS = $(AM_CPPFLAGS) $(LIBEVENT_CPPFLAGS) +libthriftz_la_CPPFLAGS = $(AM_CPPFLAGS) $(ZLIB_CPPFLAGS) + + +include_thriftdir = $(includedir)/thrift +include_thrift_HEADERS = \ + $(top_builddir)/config.h \ + src/Thrift.h \ + src/TReflectionLocal.h \ + src/TProcessor.h \ + src/TLogging.h + +include_concurrencydir = $(include_thriftdir)/concurrency +include_concurrency_HEADERS = \ + src/concurrency/Exception.h \ + src/concurrency/Mutex.h \ + src/concurrency/Monitor.h \ + src/concurrency/PosixThreadFactory.h \ + src/concurrency/Thread.h \ + src/concurrency/ThreadManager.h \ + src/concurrency/TimerManager.h \ + src/concurrency/FunctionRunner.h \ + src/concurrency/Util.h + +include_protocoldir = $(include_thriftdir)/protocol +include_protocol_HEADERS = \ + src/protocol/TBinaryProtocol.h \ + src/protocol/TCompactProtocol.h \ + src/protocol/TDenseProtocol.h \ + src/protocol/TDebugProtocol.h \ + src/protocol/TOneWayProtocol.h \ + src/protocol/TBase64Utils.h \ + src/protocol/TJSONProtocol.h \ + src/protocol/TProtocolTap.h \ + src/protocol/TProtocolException.h \ + src/protocol/TProtocol.h + +include_transportdir = $(include_thriftdir)/transport +include_transport_HEADERS = \ + src/transport/TFDTransport.h \ + src/transport/TFileTransport.h \ + src/transport/TSimpleFileTransport.h \ + src/transport/TServerSocket.h \ + src/transport/TServerTransport.h \ + src/transport/THttpClient.h \ + src/transport/TSocket.h \ + src/transport/TSocketPool.h \ + src/transport/TTransport.h \ + src/transport/TTransportException.h \ + src/transport/TTransportUtils.h \ + src/transport/TBufferTransports.h \ + src/transport/TShortReadTransport.h \ + src/transport/TZlibTransport.h + +include_serverdir = $(include_thriftdir)/server +include_server_HEADERS = \ + src/server/TServer.h \ + src/server/TSimpleServer.h \ + src/server/TThreadPoolServer.h \ + src/server/TThreadedServer.h \ + src/server/TNonblockingServer.h + +include_processordir = $(include_thriftdir)/processor +include_processor_HEADERS = \ + src/processor/PeekProcessor.h \ + src/processor/StatsProcessor.h + +noinst_PROGRAMS = concurrency_test + +concurrency_test_SOURCES = src/concurrency/test/Tests.cpp \ + src/concurrency/test/ThreadFactoryTests.h \ + src/concurrency/test/ThreadManagerTests.h \ + src/concurrency/test/TimerManagerTests.h + +concurrency_test_LDADD = libthrift.la + +EXTRA_DIST = \ + README \ + thrift-nb.pc.in \ + thrift.pc.in \ + thrift-z.pc.in diff --git a/lib/cpp/README b/lib/cpp/README new file mode 100644 index 00000000..576d0170 --- /dev/null +++ b/lib/cpp/README @@ -0,0 +1,67 @@ +Thrift C++ Software Library + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Using Thrift with C++ +===================== + +The Thrift C++ libraries are built using the GNU tools. Follow the instructions +in the top-level README, or run bootstrap.sh in this folder to generate the +Makefiles. + +In case you do not want to open another README file, do this: + ./bootstrap.sh + ./configure (--with-boost=/usr/local) + make + sudo make install + +Thrift is divided into two libraries. + +libthrift + The core Thrift library contains all the core Thrift code. It requires + boost shared pointers, pthreads, and librt. + +libthriftnb + This library contains the Thrift nonblocking server, which uses libevent. + To link this library you will also need to link libevent. + +Linking Against Thrift +====================== + +After you build and install Thrift the libraries are installed to +/usr/local/lib by default. Make sure this is in your LDPATH. + +On Linux, the best way to do this is to ensure that /usr/local/lib is in +your /etc/ld.so.conf and then run /sbin/ldconfig. + +Depending upon whether you are linking dynamically or statically and how +your build environment it set up, you may need to include additional +libraries when linking against thrift, such as librt and/or libpthread. If +you are using libthriftnb you will also need libevent. + +Dependencies +============ + +boost shared pointers +http://www.boost.org/libs/smart_ptr/smart_ptr.htm + +libevent (for libthriftnb only) +http://monkey.org/~provos/libevent/ diff --git a/lib/cpp/src/TLogging.h b/lib/cpp/src/TLogging.h new file mode 100644 index 00000000..2df82dd7 --- /dev/null +++ b/lib/cpp/src/TLogging.h @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TLOGGING_H_ +#define _THRIFT_TLOGGING_H_ 1 + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/** + * Contains utility macros for debugging and logging. + * + */ + +#ifndef HAVE_CLOCK_GETTIME +#include +#else +#include +#endif + +#ifdef HAVE_STDINT_H +#include +#endif + +/** + * T_GLOBAL_DEBUGGING_LEVEL = 0: all debugging turned off, debug macros undefined + * T_GLOBAL_DEBUGGING_LEVEL = 1: all debugging turned on + */ +#define T_GLOBAL_DEBUGGING_LEVEL 0 + + +/** + * T_GLOBAL_LOGGING_LEVEL = 0: all logging turned off, logging macros undefined + * T_GLOBAL_LOGGING_LEVEL = 1: all logging turned on + */ +#define T_GLOBAL_LOGGING_LEVEL 1 + + +/** + * Standard wrapper around fprintf what will prefix the file name and line + * number to the line. Uses T_GLOBAL_DEBUGGING_LEVEL to control whether it is + * turned on or off. + * + * @param format_string + */ +#if T_GLOBAL_DEBUGGING_LEVEL > 0 + #define T_DEBUG(format_string,...) \ + if (T_GLOBAL_DEBUGGING_LEVEL > 0) { \ + fprintf(stderr,"[%s,%d] " #format_string " \n", __FILE__, __LINE__,##__VA_ARGS__); \ + } +#else + #define T_DEBUG(format_string,...) +#endif + + +/** + * analagous to T_DEBUG but also prints the time + * + * @param string format_string input: printf style format string + */ +#if T_GLOBAL_DEBUGGING_LEVEL > 0 + #define T_DEBUG_T(format_string,...) \ + { \ + if (T_GLOBAL_DEBUGGING_LEVEL > 0) { \ + time_t now; \ + char dbgtime[26] ; \ + time(&now); \ + ctime_r(&now, dbgtime); \ + dbgtime[24] = '\0'; \ + fprintf(stderr,"[%s,%d] [%s] " #format_string " \n", __FILE__, __LINE__,dbgtime,##__VA_ARGS__); \ + } \ + } +#else + #define T_DEBUG_T(format_string,...) +#endif + + +/** + * analagous to T_DEBUG but uses input level to determine whether or not the string + * should be logged. + * + * @param int level: specified debug level + * @param string format_string input: format string + */ +#define T_DEBUG_L(level, format_string,...) \ + if ((level) > 0) { \ + fprintf(stderr,"[%s,%d] " #format_string " \n", __FILE__, __LINE__,##__VA_ARGS__); \ + } + + +/** + * Explicit error logging. Prints time, file name and line number + * + * @param string format_string input: printf style format string + */ +#define T_ERROR(format_string,...) \ + { \ + time_t now; \ + char dbgtime[26] ; \ + time(&now); \ + ctime_r(&now, dbgtime); \ + dbgtime[24] = '\0'; \ + fprintf(stderr,"[%s,%d] [%s] ERROR: " #format_string " \n", __FILE__, __LINE__,dbgtime,##__VA_ARGS__); \ + } + + +/** + * Analagous to T_ERROR, additionally aborting the process. + * WARNING: macro calls abort(), ending program execution + * + * @param string format_string input: printf style format string + */ +#define T_ERROR_ABORT(format_string,...) \ + { \ + time_t now; \ + char dbgtime[26] ; \ + time(&now); \ + ctime_r(&now, dbgtime); \ + dbgtime[24] = '\0'; \ + fprintf(stderr,"[%s,%d] [%s] ERROR: Going to abort " #format_string " \n", __FILE__, __LINE__,dbgtime,##__VA_ARGS__); \ + exit(1); \ + } + + +/** + * Log input message + * + * @param string format_string input: printf style format string + */ +#if T_GLOBAL_LOGGING_LEVEL > 0 + #define T_LOG_OPER(format_string,...) \ + { \ + if (T_GLOBAL_LOGGING_LEVEL > 0) { \ + time_t now; \ + char dbgtime[26] ; \ + time(&now); \ + ctime_r(&now, dbgtime); \ + dbgtime[24] = '\0'; \ + fprintf(stderr,"[%s] " #format_string " \n", dbgtime,##__VA_ARGS__); \ + } \ + } +#else + #define T_LOG_OPER(format_string,...) +#endif + +#endif // #ifndef _THRIFT_TLOGGING_H_ diff --git a/lib/cpp/src/TProcessor.h b/lib/cpp/src/TProcessor.h new file mode 100644 index 00000000..f2d5279a --- /dev/null +++ b/lib/cpp/src/TProcessor.h @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TPROCESSOR_H_ +#define _THRIFT_TPROCESSOR_H_ 1 + +#include +#include +#include + +namespace apache { namespace thrift { + +/** + * A processor is a generic object that acts upon two streams of data, one + * an input and the other an output. The definition of this object is loose, + * though the typical case is for some sort of server that either generates + * responses to an input stream or forwards data from one pipe onto another. + * + */ +class TProcessor { + public: + virtual ~TProcessor() {} + + virtual bool process(boost::shared_ptr in, + boost::shared_ptr out) = 0; + + bool process(boost::shared_ptr io) { + return process(io, io); + } + + protected: + TProcessor() {} +}; + +}} // apache::thrift + +#endif // #ifndef _THRIFT_PROCESSOR_H_ diff --git a/lib/cpp/src/TReflectionLocal.h b/lib/cpp/src/TReflectionLocal.h new file mode 100644 index 00000000..e83e4753 --- /dev/null +++ b/lib/cpp/src/TReflectionLocal.h @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TREFLECTIONLOCAL_H_ +#define _THRIFT_TREFLECTIONLOCAL_H_ 1 + +#include +#include +#include + +/** + * Local Reflection is a blanket term referring to the the structure + * and generation of this particular representation of Thrift types. + * (It is called local because it cannot be serialized by Thrift). + * + */ + +namespace apache { namespace thrift { namespace reflection { namespace local { + +using apache::thrift::protocol::TType; + +// We include this many bytes of the structure's fingerprint when serializing +// a top-level structure. Long enough to make collisions unlikely, short +// enough to not significantly affect the amount of memory used. +const int FP_PREFIX_LEN = 4; + +struct FieldMeta { + int16_t tag; + bool is_optional; +}; + +struct TypeSpec { + TType ttype; + uint8_t fp_prefix[FP_PREFIX_LEN]; + + // Use an anonymous union here so we can fit two TypeSpecs in one cache line. + union { + struct { + // Use parallel arrays here for denser packing (of the arrays). + FieldMeta* metas; + TypeSpec** specs; + } tstruct; + struct { + TypeSpec *subtype1; + TypeSpec *subtype2; + } tcontainer; + }; + + // Static initialization of unions isn't really possible, + // so take the plunge and use constructors. + // Hopefully they'll be evaluated at compile time. + + TypeSpec(TType ttype) : ttype(ttype) { + std::memset(fp_prefix, 0, FP_PREFIX_LEN); + } + + TypeSpec(TType ttype, + const uint8_t* fingerprint, + FieldMeta* metas, + TypeSpec** specs) : + ttype(ttype) + { + std::memcpy(fp_prefix, fingerprint, FP_PREFIX_LEN); + tstruct.metas = metas; + tstruct.specs = specs; + } + + TypeSpec(TType ttype, TypeSpec* subtype1, TypeSpec* subtype2) : + ttype(ttype) + { + std::memset(fp_prefix, 0, FP_PREFIX_LEN); + tcontainer.subtype1 = subtype1; + tcontainer.subtype2 = subtype2; + } + +}; + +}}}} // apache::thrift::reflection::local + +#endif // #ifndef _THRIFT_TREFLECTIONLOCAL_H_ diff --git a/lib/cpp/src/Thrift.cpp b/lib/cpp/src/Thrift.cpp new file mode 100644 index 00000000..ed99205b --- /dev/null +++ b/lib/cpp/src/Thrift.cpp @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +namespace apache { namespace thrift { + +TOutput GlobalOutput; + +void TOutput::printf(const char *message, ...) { + // Try to reduce heap usage, even if printf is called rarely. + static const int STACK_BUF_SIZE = 256; + char stack_buf[STACK_BUF_SIZE]; + va_list ap; + + va_start(ap, message); + int need = vsnprintf(stack_buf, STACK_BUF_SIZE, message, ap); + va_end(ap); + + if (need < STACK_BUF_SIZE) { + f_(stack_buf); + return; + } + + char *heap_buf = (char*)malloc((need+1) * sizeof(char)); + if (heap_buf == NULL) { + // Malloc failed. We might as well print the stack buffer. + f_(stack_buf); + return; + } + + va_start(ap, message); + int rval = vsnprintf(heap_buf, need+1, message, ap); + va_end(ap); + // TODO(shigin): inform user + if (rval != -1) { + f_(heap_buf); + } + free(heap_buf); +} + +void TOutput::perror(const char *message, int errno_copy) { + std::string out = message + strerror_s(errno_copy); + f_(out.c_str()); +} + +std::string TOutput::strerror_s(int errno_copy) { +#ifndef HAVE_STRERROR_R + return "errno = " + boost::lexical_cast(errno_copy); +#else // HAVE_STRERROR_R + + char b_errbuf[1024] = { '\0' }; +#ifdef STRERROR_R_CHAR_P + char *b_error = strerror_r(errno_copy, b_errbuf, sizeof(b_errbuf)); +#else + char *b_error = b_errbuf; + int rv = strerror_r(errno_copy, b_errbuf, sizeof(b_errbuf)); + if (rv == -1) { + // strerror_r failed. omgwtfbbq. + return "XSI-compliant strerror_r() failed with errno = " + + boost::lexical_cast(errno_copy); + } +#endif + // Can anyone prove that explicit cast is probably not necessary + // to ensure that the string object is constructed before + // b_error becomes invalid? + return std::string(b_error); + +#endif // HAVE_STRERROR_R +} + +uint32_t TApplicationException::read(apache::thrift::protocol::TProtocol* iprot) { + uint32_t xfer = 0; + std::string fname; + apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + while (true) { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) { + case 1: + if (ftype == apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(message_); + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == apache::thrift::protocol::T_I32) { + int32_t type; + xfer += iprot->readI32(type); + type_ = (TApplicationExceptionType)type; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + return xfer; +} + +uint32_t TApplicationException::write(apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + xfer += oprot->writeStructBegin("TApplicationException"); + xfer += oprot->writeFieldBegin("message", apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeString(message_); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("type", apache::thrift::protocol::T_I32, 2); + xfer += oprot->writeI32(type_); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +}} // apache::thrift diff --git a/lib/cpp/src/Thrift.h b/lib/cpp/src/Thrift.h new file mode 100644 index 00000000..26d2b0fc --- /dev/null +++ b/lib/cpp/src/Thrift.h @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_THRIFT_H_ +#define _THRIFT_THRIFT_H_ 1 + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include + +#include +#ifdef HAVE_INTTYPES_H +#include +#endif +#include +#include +#include +#include +#include +#include + +#include "TLogging.h" + +namespace apache { namespace thrift { + +class TOutput { + public: + TOutput() : f_(&errorTimeWrapper) {} + + inline void setOutputFunction(void (*function)(const char *)){ + f_ = function; + } + + inline void operator()(const char *message){ + f_(message); + } + + // It is important to have a const char* overload here instead of + // just the string version, otherwise errno could be corrupted + // if there is some problem allocating memory when constructing + // the string. + void perror(const char *message, int errno_copy); + inline void perror(const std::string &message, int errno_copy) { + perror(message.c_str(), errno_copy); + } + + void printf(const char *message, ...); + + inline static void errorTimeWrapper(const char* msg) { + time_t now; + char dbgtime[25]; + time(&now); + ctime_r(&now, dbgtime); + dbgtime[24] = 0; + fprintf(stderr, "Thrift: %s %s\n", dbgtime, msg); + } + + /** Just like strerror_r but returns a C++ string object. */ + static std::string strerror_s(int errno_copy); + + private: + void (*f_)(const char *); +}; + +extern TOutput GlobalOutput; + +namespace protocol { + class TProtocol; +} + +class TException : public std::exception { + public: + TException() {} + + TException(const std::string& message) : + message_(message) {} + + virtual ~TException() throw() {} + + virtual const char* what() const throw() { + if (message_.empty()) { + return "Default TException."; + } else { + return message_.c_str(); + } + } + + protected: + std::string message_; + +}; + +class TApplicationException : public TException { + public: + + /** + * Error codes for the various types of exceptions. + */ + enum TApplicationExceptionType + { UNKNOWN = 0 + , UNKNOWN_METHOD = 1 + , INVALID_MESSAGE_TYPE = 2 + , WRONG_METHOD_NAME = 3 + , BAD_SEQUENCE_ID = 4 + , MISSING_RESULT = 5 + }; + + TApplicationException() : + TException(), + type_(UNKNOWN) {} + + TApplicationException(TApplicationExceptionType type) : + TException(), + type_(type) {} + + TApplicationException(const std::string& message) : + TException(message), + type_(UNKNOWN) {} + + TApplicationException(TApplicationExceptionType type, + const std::string& message) : + TException(message), + type_(type) {} + + virtual ~TApplicationException() throw() {} + + /** + * Returns an error code that provides information about the type of error + * that has occurred. + * + * @return Error code + */ + TApplicationExceptionType getType() { + return type_; + } + + virtual const char* what() const throw() { + if (message_.empty()) { + switch (type_) { + case UNKNOWN : return "TApplicationException: Unknown application exception"; + case UNKNOWN_METHOD : return "TApplicationException: Unknown method"; + case INVALID_MESSAGE_TYPE : return "TApplicationException: Invalid message type"; + case WRONG_METHOD_NAME : return "TApplicationException: Wrong method name"; + case BAD_SEQUENCE_ID : return "TApplicationException: Bad sequence identifier"; + case MISSING_RESULT : return "TApplicationException: Missing result"; + default : return "TApplicationException: (Invalid exception type)"; + }; + } else { + return message_.c_str(); + } + } + + uint32_t read(protocol::TProtocol* iprot); + uint32_t write(protocol::TProtocol* oprot) const; + + protected: + /** + * Error code + */ + TApplicationExceptionType type_; + +}; + + +// Forward declare this structure used by TDenseProtocol +namespace reflection { namespace local { +struct TypeSpec; +}} + + +}} // apache::thrift + +#endif // #ifndef _THRIFT_THRIFT_H_ diff --git a/lib/cpp/src/concurrency/Exception.h b/lib/cpp/src/concurrency/Exception.h new file mode 100644 index 00000000..ec466297 --- /dev/null +++ b/lib/cpp/src/concurrency/Exception.h @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_EXCEPTION_H_ +#define _THRIFT_CONCURRENCY_EXCEPTION_H_ 1 + +#include +#include + +namespace apache { namespace thrift { namespace concurrency { + +class NoSuchTaskException : public apache::thrift::TException {}; + +class UncancellableTaskException : public apache::thrift::TException {}; + +class InvalidArgumentException : public apache::thrift::TException {}; + +class IllegalStateException : public apache::thrift::TException {}; + +class TimedOutException : public apache::thrift::TException { +public: + TimedOutException():TException("TimedOutException"){}; + TimedOutException(const std::string& message ) : + TException(message) {} +}; + +class TooManyPendingTasksException : public apache::thrift::TException { +public: + TooManyPendingTasksException():TException("TooManyPendingTasksException"){}; + TooManyPendingTasksException(const std::string& message ) : + TException(message) {} +}; + +class SystemResourceException : public apache::thrift::TException { +public: + SystemResourceException() {} + + SystemResourceException(const std::string& message) : + TException(message) {} +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_EXCEPTION_H_ diff --git a/lib/cpp/src/concurrency/FunctionRunner.h b/lib/cpp/src/concurrency/FunctionRunner.h new file mode 100644 index 00000000..22169276 --- /dev/null +++ b/lib/cpp/src/concurrency/FunctionRunner.h @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_FUNCTION_RUNNER_H +#define _THRIFT_CONCURRENCY_FUNCTION_RUNNER_H 1 + +#include +#include "thrift/lib/cpp/concurrency/Thread.h" + +namespace apache { namespace thrift { namespace concurrency { + +/** + * Convenient implementation of Runnable that will execute arbitrary callbacks. + * Interfaces are provided to accept both a generic 'void(void)' callback, and + * a 'void* (void*)' pthread_create-style callback. + * + * Example use: + * void* my_thread_main(void* arg); + * shared_ptr factory = ...; + * shared_ptr thread = + * factory->newThread(shared_ptr( + * new FunctionRunner(my_thread_main, some_argument))); + * thread->start(); + * + * + */ + +class FunctionRunner : public Runnable { + public: + // This is the type of callback 'pthread_create()' expects. + typedef void* (*PthreadFuncPtr)(void *arg); + // This a fully-generic void(void) callback for custom bindings. + typedef std::tr1::function VoidFunc; + + /** + * Given a 'pthread_create' style callback, this FunctionRunner will + * execute the given callback. Note that the 'void*' return value is ignored. + */ + FunctionRunner(PthreadFuncPtr func, void* arg) + : func_(std::tr1::bind(func, arg)) + { } + + /** + * Given a generic callback, this FunctionRunner will execute it. + */ + FunctionRunner(const VoidFunc& cob) + : func_(cob) + { } + + + void run() { + func_(); + } + + private: + VoidFunc func_; +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_FUNCTION_RUNNER_H diff --git a/lib/cpp/src/concurrency/Monitor.cpp b/lib/cpp/src/concurrency/Monitor.cpp new file mode 100644 index 00000000..2055caa9 --- /dev/null +++ b/lib/cpp/src/concurrency/Monitor.cpp @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "Monitor.h" +#include "Exception.h" +#include "Util.h" + +#include +#include + +#include + +#include + +namespace apache { namespace thrift { namespace concurrency { + +/** + * Monitor implementation using the POSIX pthread library + * + * @version $Id:$ + */ +class Monitor::Impl { + + public: + + Impl() : + mutexInitialized_(false), + condInitialized_(false) { + + if (pthread_mutex_init(&pthread_mutex_, NULL) == 0) { + mutexInitialized_ = true; + + if (pthread_cond_init(&pthread_cond_, NULL) == 0) { + condInitialized_ = true; + } + } + + if (!mutexInitialized_ || !condInitialized_) { + cleanup(); + throw SystemResourceException(); + } + } + + ~Impl() { cleanup(); } + + void lock() const { pthread_mutex_lock(&pthread_mutex_); } + + void unlock() const { pthread_mutex_unlock(&pthread_mutex_); } + + void wait(int64_t timeout) const { + + // XXX Need to assert that caller owns mutex + assert(timeout >= 0LL); + if (timeout == 0LL) { + int iret = pthread_cond_wait(&pthread_cond_, &pthread_mutex_); + assert(iret == 0); + } else { + struct timespec abstime; + int64_t now = Util::currentTime(); + Util::toTimespec(abstime, now + timeout); + int result = pthread_cond_timedwait(&pthread_cond_, + &pthread_mutex_, + &abstime); + if (result == ETIMEDOUT) { + // pthread_cond_timedwait has been observed to return early on + // various platforms, so comment out this assert. + //assert(Util::currentTime() >= (now + timeout)); + throw TimedOutException(); + } + } + } + + void notify() { + // XXX Need to assert that caller owns mutex + int iret = pthread_cond_signal(&pthread_cond_); + assert(iret == 0); + } + + void notifyAll() { + // XXX Need to assert that caller owns mutex + int iret = pthread_cond_broadcast(&pthread_cond_); + assert(iret == 0); + } + + private: + + void cleanup() { + if (mutexInitialized_) { + mutexInitialized_ = false; + int iret = pthread_mutex_destroy(&pthread_mutex_); + assert(iret == 0); + } + + if (condInitialized_) { + condInitialized_ = false; + int iret = pthread_cond_destroy(&pthread_cond_); + assert(iret == 0); + } + } + + mutable pthread_mutex_t pthread_mutex_; + mutable bool mutexInitialized_; + mutable pthread_cond_t pthread_cond_; + mutable bool condInitialized_; +}; + +Monitor::Monitor() : impl_(new Monitor::Impl()) {} + +Monitor::~Monitor() { delete impl_; } + +void Monitor::lock() const { impl_->lock(); } + +void Monitor::unlock() const { impl_->unlock(); } + +void Monitor::wait(int64_t timeout) const { impl_->wait(timeout); } + +void Monitor::notify() const { impl_->notify(); } + +void Monitor::notifyAll() const { impl_->notifyAll(); } + +}}} // apache::thrift::concurrency diff --git a/lib/cpp/src/concurrency/Monitor.h b/lib/cpp/src/concurrency/Monitor.h new file mode 100644 index 00000000..234bf326 --- /dev/null +++ b/lib/cpp/src/concurrency/Monitor.h @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_MONITOR_H_ +#define _THRIFT_CONCURRENCY_MONITOR_H_ 1 + +#include "Exception.h" + +namespace apache { namespace thrift { namespace concurrency { + +/** + * A monitor is a combination mutex and condition-event. Waiting and + * notifying condition events requires that the caller own the mutex. Mutex + * lock and unlock operations can be performed independently of condition + * events. This is more or less analogous to java.lang.Object multi-thread + * operations + * + * Note that all methods are const. Monitors implement logical constness, not + * bit constness. This allows const methods to call monitor methods without + * needing to cast away constness or change to non-const signatures. + * + * @version $Id:$ + */ +class Monitor { + + public: + + Monitor(); + + virtual ~Monitor(); + + virtual void lock() const; + + virtual void unlock() const; + + virtual void wait(int64_t timeout=0LL) const; + + virtual void notify() const; + + virtual void notifyAll() const; + + private: + + class Impl; + + Impl* impl_; +}; + +class Synchronized { + public: + + Synchronized(const Monitor& value) : + monitor_(value) { + monitor_.lock(); + } + + ~Synchronized() { + monitor_.unlock(); + } + + private: + const Monitor& monitor_; +}; + + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_MONITOR_H_ diff --git a/lib/cpp/src/concurrency/Mutex.cpp b/lib/cpp/src/concurrency/Mutex.cpp new file mode 100644 index 00000000..045dbdfe --- /dev/null +++ b/lib/cpp/src/concurrency/Mutex.cpp @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "Mutex.h" + +#include +#include + +using boost::shared_ptr; + +namespace apache { namespace thrift { namespace concurrency { + +/** + * Implementation of Mutex class using POSIX mutex + * + * @version $Id:$ + */ +class Mutex::impl { + public: + impl(Initializer init) : initialized_(false) { + init(&pthread_mutex_); + initialized_ = true; + } + + ~impl() { + if (initialized_) { + initialized_ = false; + int ret = pthread_mutex_destroy(&pthread_mutex_); + assert(ret == 0); + } + } + + void lock() const { pthread_mutex_lock(&pthread_mutex_); } + + bool trylock() const { return (0 == pthread_mutex_trylock(&pthread_mutex_)); } + + void unlock() const { pthread_mutex_unlock(&pthread_mutex_); } + + private: + mutable pthread_mutex_t pthread_mutex_; + mutable bool initialized_; +}; + +Mutex::Mutex(Initializer init) : impl_(new Mutex::impl(init)) {} + +void Mutex::lock() const { impl_->lock(); } + +bool Mutex::trylock() const { return impl_->trylock(); } + +void Mutex::unlock() const { impl_->unlock(); } + +void Mutex::DEFAULT_INITIALIZER(void* arg) { + pthread_mutex_t* pthread_mutex = (pthread_mutex_t*)arg; + int ret = pthread_mutex_init(pthread_mutex, NULL); + assert(ret == 0); +} + +static void init_with_kind(pthread_mutex_t* mutex, int kind) { + pthread_mutexattr_t mutexattr; + int ret = pthread_mutexattr_init(&mutexattr); + assert(ret == 0); + + // Apparently, this can fail. Should we really be aborting? + ret = pthread_mutexattr_settype(&mutexattr, kind); + assert(ret == 0); + + ret = pthread_mutex_init(mutex, &mutexattr); + assert(ret == 0); + + ret = pthread_mutexattr_destroy(&mutexattr); + assert(ret == 0); +} + +#ifdef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP +void Mutex::ADAPTIVE_INITIALIZER(void* arg) { + // From mysql source: mysys/my_thr_init.c + // Set mutex type to "fast" a.k.a "adaptive" + // + // In this case the thread may steal the mutex from some other thread + // that is waiting for the same mutex. This will save us some + // context switches but may cause a thread to 'starve forever' while + // waiting for the mutex (not likely if the code within the mutex is + // short). + init_with_kind((pthread_mutex_t*)arg, PTHREAD_MUTEX_ADAPTIVE_NP); +} +#endif + +#ifdef PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP +void Mutex::RECURSIVE_INITIALIZER(void* arg) { + init_with_kind((pthread_mutex_t*)arg, PTHREAD_MUTEX_RECURSIVE_NP); +} +#endif + + +/** + * Implementation of ReadWriteMutex class using POSIX rw lock + * + * @version $Id:$ + */ +class ReadWriteMutex::impl { +public: + impl() : initialized_(false) { + int ret = pthread_rwlock_init(&rw_lock_, NULL); + assert(ret == 0); + initialized_ = true; + } + + ~impl() { + if(initialized_) { + initialized_ = false; + int ret = pthread_rwlock_destroy(&rw_lock_); + assert(ret == 0); + } + } + + void acquireRead() const { pthread_rwlock_rdlock(&rw_lock_); } + + void acquireWrite() const { pthread_rwlock_wrlock(&rw_lock_); } + + bool attemptRead() const { return pthread_rwlock_tryrdlock(&rw_lock_); } + + bool attemptWrite() const { return pthread_rwlock_trywrlock(&rw_lock_); } + + void release() const { pthread_rwlock_unlock(&rw_lock_); } + +private: + mutable pthread_rwlock_t rw_lock_; + mutable bool initialized_; +}; + +ReadWriteMutex::ReadWriteMutex() : impl_(new ReadWriteMutex::impl()) {} + +void ReadWriteMutex::acquireRead() const { impl_->acquireRead(); } + +void ReadWriteMutex::acquireWrite() const { impl_->acquireWrite(); } + +bool ReadWriteMutex::attemptRead() const { return impl_->attemptRead(); } + +bool ReadWriteMutex::attemptWrite() const { return impl_->attemptWrite(); } + +void ReadWriteMutex::release() const { impl_->release(); } + +}}} // apache::thrift::concurrency + diff --git a/lib/cpp/src/concurrency/Mutex.h b/lib/cpp/src/concurrency/Mutex.h new file mode 100644 index 00000000..884412be --- /dev/null +++ b/lib/cpp/src/concurrency/Mutex.h @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_MUTEX_H_ +#define _THRIFT_CONCURRENCY_MUTEX_H_ 1 + +#include + +namespace apache { namespace thrift { namespace concurrency { + +/** + * A simple mutex class + * + * @version $Id:$ + */ +class Mutex { + public: + typedef void (*Initializer)(void*); + + Mutex(Initializer init = DEFAULT_INITIALIZER); + virtual ~Mutex() {} + virtual void lock() const; + virtual bool trylock() const; + virtual void unlock() const; + + static void DEFAULT_INITIALIZER(void*); + static void ADAPTIVE_INITIALIZER(void*); + static void RECURSIVE_INITIALIZER(void*); + + private: + + class impl; + boost::shared_ptr impl_; +}; + +class ReadWriteMutex { +public: + ReadWriteMutex(); + virtual ~ReadWriteMutex() {} + + // these get the lock and block until it is done successfully + virtual void acquireRead() const; + virtual void acquireWrite() const; + + // these attempt to get the lock, returning false immediately if they fail + virtual bool attemptRead() const; + virtual bool attemptWrite() const; + + // this releases both read and write locks + virtual void release() const; + +private: + + class impl; + boost::shared_ptr impl_; +}; + +class Guard { + public: + Guard(const Mutex& value) : mutex_(value) { + mutex_.lock(); + } + ~Guard() { + mutex_.unlock(); + } + + private: + const Mutex& mutex_; +}; + +class RWGuard { + public: + RWGuard(const ReadWriteMutex& value, bool write = 0) : rw_mutex_(value) { + if (write) { + rw_mutex_.acquireWrite(); + } else { + rw_mutex_.acquireRead(); + } + } + ~RWGuard() { + rw_mutex_.release(); + } + private: + const ReadWriteMutex& rw_mutex_; +}; + + +// A little hack to prevent someone from trying to do "Guard(m);" +// Sorry for polluting the global namespace, but I think it's worth it. +#define Guard(m) incorrect_use_of_Guard(m) +#define RWGuard(m) incorrect_use_of_RWGuard(m) + + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_MUTEX_H_ diff --git a/lib/cpp/src/concurrency/PosixThreadFactory.cpp b/lib/cpp/src/concurrency/PosixThreadFactory.cpp new file mode 100644 index 00000000..e48dce39 --- /dev/null +++ b/lib/cpp/src/concurrency/PosixThreadFactory.cpp @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "PosixThreadFactory.h" +#include "Exception.h" + +#if GOOGLE_PERFTOOLS_REGISTER_THREAD +# include +#endif + +#include +#include + +#include + +#include + +namespace apache { namespace thrift { namespace concurrency { + +using boost::shared_ptr; +using boost::weak_ptr; + +/** + * The POSIX thread class. + * + * @version $Id:$ + */ +class PthreadThread: public Thread { + public: + + enum STATE { + uninitialized, + starting, + started, + stopping, + stopped + }; + + static const int MB = 1024 * 1024; + + static void* threadMain(void* arg); + + private: + pthread_t pthread_; + STATE state_; + int policy_; + int priority_; + int stackSize_; + weak_ptr self_; + bool detached_; + + public: + + PthreadThread(int policy, int priority, int stackSize, bool detached, shared_ptr runnable) : + pthread_(0), + state_(uninitialized), + policy_(policy), + priority_(priority), + stackSize_(stackSize), + detached_(detached) { + + this->Thread::runnable(runnable); + } + + ~PthreadThread() { + /* Nothing references this thread, if is is not detached, do a join + now, otherwise the thread-id and, possibly, other resources will + be leaked. */ + if(!detached_) { + try { + join(); + } catch(...) { + // We're really hosed. + } + } + } + + void start() { + if (state_ != uninitialized) { + return; + } + + pthread_attr_t thread_attr; + if (pthread_attr_init(&thread_attr) != 0) { + throw SystemResourceException("pthread_attr_init failed"); + } + + if(pthread_attr_setdetachstate(&thread_attr, + detached_ ? + PTHREAD_CREATE_DETACHED : + PTHREAD_CREATE_JOINABLE) != 0) { + throw SystemResourceException("pthread_attr_setdetachstate failed"); + } + + // Set thread stack size + if (pthread_attr_setstacksize(&thread_attr, MB * stackSize_) != 0) { + throw SystemResourceException("pthread_attr_setstacksize failed"); + } + + // Set thread policy + if (pthread_attr_setschedpolicy(&thread_attr, policy_) != 0) { + throw SystemResourceException("pthread_attr_setschedpolicy failed"); + } + + struct sched_param sched_param; + sched_param.sched_priority = priority_; + + // Set thread priority + if (pthread_attr_setschedparam(&thread_attr, &sched_param) != 0) { + throw SystemResourceException("pthread_attr_setschedparam failed"); + } + + // Create reference + shared_ptr* selfRef = new shared_ptr(); + *selfRef = self_.lock(); + + state_ = starting; + + if (pthread_create(&pthread_, &thread_attr, threadMain, (void*)selfRef) != 0) { + throw SystemResourceException("pthread_create failed"); + } + } + + void join() { + if (!detached_ && state_ != uninitialized) { + void* ignore; + /* XXX + If join fails it is most likely due to the fact + that the last reference was the thread itself and cannot + join. This results in leaked threads and will eventually + cause the process to run out of thread resources. + We're beyond the point of throwing an exception. Not clear how + best to handle this. */ + detached_ = pthread_join(pthread_, &ignore) == 0; + } + } + + Thread::id_t getId() { + return (Thread::id_t)pthread_; + } + + shared_ptr runnable() const { return Thread::runnable(); } + + void runnable(shared_ptr value) { Thread::runnable(value); } + + void weakRef(shared_ptr self) { + assert(self.get() == this); + self_ = weak_ptr(self); + } +}; + +void* PthreadThread::threadMain(void* arg) { + shared_ptr thread = *(shared_ptr*)arg; + delete reinterpret_cast*>(arg); + + if (thread == NULL) { + return (void*)0; + } + + if (thread->state_ != starting) { + return (void*)0; + } + +#if GOOGLE_PERFTOOLS_REGISTER_THREAD + ProfilerRegisterThread(); +#endif + + thread->state_ = starting; + thread->runnable()->run(); + if (thread->state_ != stopping && thread->state_ != stopped) { + thread->state_ = stopping; + } + + return (void*)0; +} + +/** + * POSIX Thread factory implementation + */ +class PosixThreadFactory::Impl { + + private: + POLICY policy_; + PRIORITY priority_; + int stackSize_; + bool detached_; + + /** + * Converts generic posix thread schedule policy enums into pthread + * API values. + */ + static int toPthreadPolicy(POLICY policy) { + switch (policy) { + case OTHER: + return SCHED_OTHER; + case FIFO: + return SCHED_FIFO; + case ROUND_ROBIN: + return SCHED_RR; + } + return SCHED_OTHER; + } + + /** + * Converts relative thread priorities to absolute value based on posix + * thread scheduler policy + * + * The idea is simply to divide up the priority range for the given policy + * into the correpsonding relative priority level (lowest..highest) and + * then pro-rate accordingly. + */ + static int toPthreadPriority(POLICY policy, PRIORITY priority) { + int pthread_policy = toPthreadPolicy(policy); + int min_priority = sched_get_priority_min(pthread_policy); + int max_priority = sched_get_priority_max(pthread_policy); + int quanta = (HIGHEST - LOWEST) + 1; + float stepsperquanta = (max_priority - min_priority) / quanta; + + if (priority <= HIGHEST) { + return (int)(min_priority + stepsperquanta * priority); + } else { + // should never get here for priority increments. + assert(false); + return (int)(min_priority + stepsperquanta * NORMAL); + } + } + + public: + + Impl(POLICY policy, PRIORITY priority, int stackSize, bool detached) : + policy_(policy), + priority_(priority), + stackSize_(stackSize), + detached_(detached) {} + + /** + * Creates a new POSIX thread to run the runnable object + * + * @param runnable A runnable object + */ + shared_ptr newThread(shared_ptr runnable) const { + shared_ptr result = shared_ptr(new PthreadThread(toPthreadPolicy(policy_), toPthreadPriority(policy_, priority_), stackSize_, detached_, runnable)); + result->weakRef(result); + runnable->thread(result); + return result; + } + + int getStackSize() const { return stackSize_; } + + void setStackSize(int value) { stackSize_ = value; } + + PRIORITY getPriority() const { return priority_; } + + /** + * Sets priority. + * + * XXX + * Need to handle incremental priorities properly. + */ + void setPriority(PRIORITY value) { priority_ = value; } + + bool isDetached() const { return detached_; } + + void setDetached(bool value) { detached_ = value; } + + Thread::id_t getCurrentThreadId() const { + // TODO(dreiss): Stop using C-style casts. + return (id_t)pthread_self(); + } + +}; + +PosixThreadFactory::PosixThreadFactory(POLICY policy, PRIORITY priority, int stackSize, bool detached) : + impl_(new PosixThreadFactory::Impl(policy, priority, stackSize, detached)) {} + +shared_ptr PosixThreadFactory::newThread(shared_ptr runnable) const { return impl_->newThread(runnable); } + +int PosixThreadFactory::getStackSize() const { return impl_->getStackSize(); } + +void PosixThreadFactory::setStackSize(int value) { impl_->setStackSize(value); } + +PosixThreadFactory::PRIORITY PosixThreadFactory::getPriority() const { return impl_->getPriority(); } + +void PosixThreadFactory::setPriority(PosixThreadFactory::PRIORITY value) { impl_->setPriority(value); } + +bool PosixThreadFactory::isDetached() const { return impl_->isDetached(); } + +void PosixThreadFactory::setDetached(bool value) { impl_->setDetached(value); } + +Thread::id_t PosixThreadFactory::getCurrentThreadId() const { return impl_->getCurrentThreadId(); } + +}}} // apache::thrift::concurrency diff --git a/lib/cpp/src/concurrency/PosixThreadFactory.h b/lib/cpp/src/concurrency/PosixThreadFactory.h new file mode 100644 index 00000000..d6d83a3a --- /dev/null +++ b/lib/cpp/src/concurrency/PosixThreadFactory.h @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_POSIXTHREADFACTORY_H_ +#define _THRIFT_CONCURRENCY_POSIXTHREADFACTORY_H_ 1 + +#include "Thread.h" + +#include + +namespace apache { namespace thrift { namespace concurrency { + +/** + * A thread factory to create posix threads + * + * @version $Id:$ + */ +class PosixThreadFactory : public ThreadFactory { + + public: + + /** + * POSIX Thread scheduler policies + */ + enum POLICY { + OTHER, + FIFO, + ROUND_ROBIN + }; + + /** + * POSIX Thread scheduler relative priorities, + * + * Absolute priority is determined by scheduler policy and OS. This + * enumeration specifies relative priorities such that one can specify a + * priority withing a giving scheduler policy without knowing the absolute + * value of the priority. + */ + enum PRIORITY { + LOWEST = 0, + LOWER = 1, + LOW = 2, + NORMAL = 3, + HIGH = 4, + HIGHER = 5, + HIGHEST = 6, + INCREMENT = 7, + DECREMENT = 8 + }; + + /** + * Posix thread (pthread) factory. All threads created by a factory are reference-counted + * via boost::shared_ptr and boost::weak_ptr. The factory guarantees that threads and + * the Runnable tasks they host will be properly cleaned up once the last strong reference + * to both is given up. + * + * Threads are created with the specified policy, priority, stack-size and detachable-mode + * detached means the thread is free-running and will release all system resources the + * when it completes. A detachable thread is not joinable. The join method + * of a detachable thread will return immediately with no error. + * + * By default threads are not joinable. + */ + + PosixThreadFactory(POLICY policy=ROUND_ROBIN, PRIORITY priority=NORMAL, int stackSize=1, bool detached=true); + + // From ThreadFactory; + boost::shared_ptr newThread(boost::shared_ptr runnable) const; + + // From ThreadFactory; + Thread::id_t getCurrentThreadId() const; + + /** + * Gets stack size for created threads + * + * @return int size in megabytes + */ + virtual int getStackSize() const; + + /** + * Sets stack size for created threads + * + * @param value size in megabytes + */ + virtual void setStackSize(int value); + + /** + * Gets priority relative to current policy + */ + virtual PRIORITY getPriority() const; + + /** + * Sets priority relative to current policy + */ + virtual void setPriority(PRIORITY priority); + + /** + * Sets detached mode of threads + */ + virtual void setDetached(bool detached); + + /** + * Gets current detached mode + */ + virtual bool isDetached() const; + + private: + class Impl; + boost::shared_ptr impl_; +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_POSIXTHREADFACTORY_H_ diff --git a/lib/cpp/src/concurrency/Thread.h b/lib/cpp/src/concurrency/Thread.h new file mode 100644 index 00000000..d4282adb --- /dev/null +++ b/lib/cpp/src/concurrency/Thread.h @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_THREAD_H_ +#define _THRIFT_CONCURRENCY_THREAD_H_ 1 + +#include +#include +#include + +namespace apache { namespace thrift { namespace concurrency { + +class Thread; + +/** + * Minimal runnable class. More or less analogous to java.lang.Runnable. + * + * @version $Id:$ + */ +class Runnable { + + public: + virtual ~Runnable() {}; + virtual void run() = 0; + + /** + * Gets the thread object that is hosting this runnable object - can return + * an empty boost::shared pointer if no references remain on thet thread object + */ + virtual boost::shared_ptr thread() { return thread_.lock(); } + + /** + * Sets the thread that is executing this object. This is only meant for + * use by concrete implementations of Thread. + */ + virtual void thread(boost::shared_ptr value) { thread_ = value; } + + private: + boost::weak_ptr thread_; +}; + +/** + * Minimal thread class. Returned by thread factory bound to a Runnable object + * and ready to start execution. More or less analogous to java.lang.Thread + * (minus all the thread group, priority, mode and other baggage, since that + * is difficult to abstract across platforms and is left for platform-specific + * ThreadFactory implemtations to deal with + * + * @see apache::thrift::concurrency::ThreadFactory) + */ +class Thread { + + public: + + typedef uint64_t id_t; + + virtual ~Thread() {}; + + /** + * Starts the thread. Does platform specific thread creation and + * configuration then invokes the run method of the Runnable object bound + * to this thread. + */ + virtual void start() = 0; + + /** + * Join this thread. Current thread blocks until this target thread + * completes. + */ + virtual void join() = 0; + + /** + * Gets the thread's platform-specific ID + */ + virtual id_t getId() = 0; + + /** + * Gets the runnable object this thread is hosting + */ + virtual boost::shared_ptr runnable() const { return _runnable; } + + protected: + virtual void runnable(boost::shared_ptr value) { _runnable = value; } + + private: + boost::shared_ptr _runnable; + +}; + +/** + * Factory to create platform-specific thread object and bind them to Runnable + * object for execution + */ +class ThreadFactory { + + public: + virtual ~ThreadFactory() {} + virtual boost::shared_ptr newThread(boost::shared_ptr runnable) const = 0; + + /** Gets the current thread id or unknown_thread_id if the current thread is not a thrift thread */ + + static const Thread::id_t unknown_thread_id; + + virtual Thread::id_t getCurrentThreadId() const = 0; +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_THREAD_H_ diff --git a/lib/cpp/src/concurrency/ThreadManager.cpp b/lib/cpp/src/concurrency/ThreadManager.cpp new file mode 100644 index 00000000..abfcf6e7 --- /dev/null +++ b/lib/cpp/src/concurrency/ThreadManager.cpp @@ -0,0 +1,493 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "ThreadManager.h" +#include "Exception.h" +#include "Monitor.h" + +#include + +#include +#include +#include + +#if defined(DEBUG) +#include +#endif //defined(DEBUG) + +namespace apache { namespace thrift { namespace concurrency { + +using boost::shared_ptr; +using boost::dynamic_pointer_cast; + +/** + * ThreadManager class + * + * This class manages a pool of threads. It uses a ThreadFactory to create + * threads. It never actually creates or destroys worker threads, rather + * it maintains statistics on number of idle threads, number of active threads, + * task backlog, and average wait and service times. + * + * @version $Id:$ + */ +class ThreadManager::Impl : public ThreadManager { + + public: + Impl() : + workerCount_(0), + workerMaxCount_(0), + idleCount_(0), + pendingTaskCountMax_(0), + state_(ThreadManager::UNINITIALIZED) {} + + ~Impl() { stop(); } + + void start(); + + void stop() { stopImpl(false); } + + void join() { stopImpl(true); } + + const ThreadManager::STATE state() const { + return state_; + } + + shared_ptr threadFactory() const { + Synchronized s(monitor_); + return threadFactory_; + } + + void threadFactory(shared_ptr value) { + Synchronized s(monitor_); + threadFactory_ = value; + } + + void addWorker(size_t value); + + void removeWorker(size_t value); + + size_t idleWorkerCount() const { + return idleCount_; + } + + size_t workerCount() const { + Synchronized s(monitor_); + return workerCount_; + } + + size_t pendingTaskCount() const { + Synchronized s(monitor_); + return tasks_.size(); + } + + size_t totalTaskCount() const { + Synchronized s(monitor_); + return tasks_.size() + workerCount_ - idleCount_; + } + + size_t pendingTaskCountMax() const { + Synchronized s(monitor_); + return pendingTaskCountMax_; + } + + void pendingTaskCountMax(const size_t value) { + Synchronized s(monitor_); + pendingTaskCountMax_ = value; + } + + bool canSleep(); + + void add(shared_ptr value, int64_t timeout); + + void remove(shared_ptr task); + +private: + void stopImpl(bool join); + + size_t workerCount_; + size_t workerMaxCount_; + size_t idleCount_; + size_t pendingTaskCountMax_; + + ThreadManager::STATE state_; + shared_ptr threadFactory_; + + + friend class ThreadManager::Task; + std::queue > tasks_; + Monitor monitor_; + Monitor workerMonitor_; + + friend class ThreadManager::Worker; + std::set > workers_; + std::set > deadWorkers_; + std::map > idMap_; +}; + +class ThreadManager::Task : public Runnable { + + public: + enum STATE { + WAITING, + EXECUTING, + CANCELLED, + COMPLETE + }; + + Task(shared_ptr runnable) : + runnable_(runnable), + state_(WAITING) {} + + ~Task() {} + + void run() { + if (state_ == EXECUTING) { + runnable_->run(); + state_ = COMPLETE; + } + } + + private: + shared_ptr runnable_; + friend class ThreadManager::Worker; + STATE state_; +}; + +class ThreadManager::Worker: public Runnable { + enum STATE { + UNINITIALIZED, + STARTING, + STARTED, + STOPPING, + STOPPED + }; + + public: + Worker(ThreadManager::Impl* manager) : + manager_(manager), + state_(UNINITIALIZED), + idle_(false) {} + + ~Worker() {} + + private: + bool isActive() const { + return + (manager_->workerCount_ <= manager_->workerMaxCount_) || + (manager_->state_ == JOINING && !manager_->tasks_.empty()); + } + + public: + /** + * Worker entry point + * + * As long as worker thread is running, pull tasks off the task queue and + * execute. + */ + void run() { + bool active = false; + bool notifyManager = false; + + /** + * Increment worker semaphore and notify manager if worker count reached + * desired max + * + * Note: We have to release the monitor and acquire the workerMonitor + * since that is what the manager blocks on for worker add/remove + */ + { + Synchronized s(manager_->monitor_); + active = manager_->workerCount_ < manager_->workerMaxCount_; + if (active) { + manager_->workerCount_++; + notifyManager = manager_->workerCount_ == manager_->workerMaxCount_; + } + } + + if (notifyManager) { + Synchronized s(manager_->workerMonitor_); + manager_->workerMonitor_.notify(); + notifyManager = false; + } + + while (active) { + shared_ptr task; + + /** + * While holding manager monitor block for non-empty task queue (Also + * check that the thread hasn't been requested to stop). Once the queue + * is non-empty, dequeue a task, release monitor, and execute. If the + * worker max count has been decremented such that we exceed it, mark + * ourself inactive, decrement the worker count and notify the manager + * (technically we're notifying the next blocked thread but eventually + * the manager will see it. + */ + { + Synchronized s(manager_->monitor_); + active = isActive(); + + while (active && manager_->tasks_.empty()) { + manager_->idleCount_++; + idle_ = true; + manager_->monitor_.wait(); + active = isActive(); + idle_ = false; + manager_->idleCount_--; + } + + if (active) { + if (!manager_->tasks_.empty()) { + task = manager_->tasks_.front(); + manager_->tasks_.pop(); + if (task->state_ == ThreadManager::Task::WAITING) { + task->state_ = ThreadManager::Task::EXECUTING; + } + + /* If we have a pending task max and we just dropped below it, wakeup any + thread that might be blocked on add. */ + if (manager_->pendingTaskCountMax_ != 0 && + manager_->tasks_.size() == manager_->pendingTaskCountMax_ - 1) { + manager_->monitor_.notify(); + } + } + } else { + idle_ = true; + manager_->workerCount_--; + notifyManager = (manager_->workerCount_ == manager_->workerMaxCount_); + } + } + + if (task != NULL) { + if (task->state_ == ThreadManager::Task::EXECUTING) { + try { + task->run(); + } catch(...) { + // XXX need to log this + } + } + } + } + + { + Synchronized s(manager_->workerMonitor_); + manager_->deadWorkers_.insert(this->thread()); + if (notifyManager) { + manager_->workerMonitor_.notify(); + } + } + + return; + } + + private: + ThreadManager::Impl* manager_; + friend class ThreadManager::Impl; + STATE state_; + bool idle_; +}; + + + void ThreadManager::Impl::addWorker(size_t value) { + std::set > newThreads; + for (size_t ix = 0; ix < value; ix++) { + class ThreadManager::Worker; + shared_ptr worker = shared_ptr(new ThreadManager::Worker(this)); + newThreads.insert(threadFactory_->newThread(worker)); + } + + { + Synchronized s(monitor_); + workerMaxCount_ += value; + workers_.insert(newThreads.begin(), newThreads.end()); + } + + for (std::set >::iterator ix = newThreads.begin(); ix != newThreads.end(); ix++) { + shared_ptr worker = dynamic_pointer_cast((*ix)->runnable()); + worker->state_ = ThreadManager::Worker::STARTING; + (*ix)->start(); + idMap_.insert(std::pair >((*ix)->getId(), *ix)); + } + + { + Synchronized s(workerMonitor_); + while (workerCount_ != workerMaxCount_) { + workerMonitor_.wait(); + } + } +} + +void ThreadManager::Impl::start() { + + if (state_ == ThreadManager::STOPPED) { + return; + } + + { + Synchronized s(monitor_); + if (state_ == ThreadManager::UNINITIALIZED) { + if (threadFactory_ == NULL) { + throw InvalidArgumentException(); + } + state_ = ThreadManager::STARTED; + monitor_.notifyAll(); + } + + while (state_ == STARTING) { + monitor_.wait(); + } + } +} + +void ThreadManager::Impl::stopImpl(bool join) { + bool doStop = false; + if (state_ == ThreadManager::STOPPED) { + return; + } + + { + Synchronized s(monitor_); + if (state_ != ThreadManager::STOPPING && + state_ != ThreadManager::JOINING && + state_ != ThreadManager::STOPPED) { + doStop = true; + state_ = join ? ThreadManager::JOINING : ThreadManager::STOPPING; + } + } + + if (doStop) { + removeWorker(workerCount_); + } + + // XXX + // should be able to block here for transition to STOPPED since we're no + // using shared_ptrs + + { + Synchronized s(monitor_); + state_ = ThreadManager::STOPPED; + } + +} + +void ThreadManager::Impl::removeWorker(size_t value) { + std::set > removedThreads; + { + Synchronized s(monitor_); + if (value > workerMaxCount_) { + throw InvalidArgumentException(); + } + + workerMaxCount_ -= value; + + if (idleCount_ < value) { + for (size_t ix = 0; ix < idleCount_; ix++) { + monitor_.notify(); + } + } else { + monitor_.notifyAll(); + } + } + + { + Synchronized s(workerMonitor_); + + while (workerCount_ != workerMaxCount_) { + workerMonitor_.wait(); + } + + for (std::set >::iterator ix = deadWorkers_.begin(); ix != deadWorkers_.end(); ix++) { + workers_.erase(*ix); + idMap_.erase((*ix)->getId()); + } + + deadWorkers_.clear(); + } +} + + bool ThreadManager::Impl::canSleep() { + const Thread::id_t id = threadFactory_->getCurrentThreadId(); + return idMap_.find(id) == idMap_.end(); + } + + void ThreadManager::Impl::add(shared_ptr value, int64_t timeout) { + Synchronized s(monitor_); + + if (state_ != ThreadManager::STARTED) { + throw IllegalStateException(); + } + + if (pendingTaskCountMax_ > 0 && (tasks_.size() >= pendingTaskCountMax_)) { + if (canSleep() && timeout >= 0) { + while (pendingTaskCountMax_ > 0 && tasks_.size() >= pendingTaskCountMax_) { + monitor_.wait(timeout); + } + } else { + throw TooManyPendingTasksException(); + } + } + + tasks_.push(shared_ptr(new ThreadManager::Task(value))); + + // If idle thread is available notify it, otherwise all worker threads are + // running and will get around to this task in time. + if (idleCount_ > 0) { + monitor_.notify(); + } + } + +void ThreadManager::Impl::remove(shared_ptr task) { + Synchronized s(monitor_); + if (state_ != ThreadManager::STARTED) { + throw IllegalStateException(); + } +} + +class SimpleThreadManager : public ThreadManager::Impl { + + public: + SimpleThreadManager(size_t workerCount=4, size_t pendingTaskCountMax=0) : + workerCount_(workerCount), + pendingTaskCountMax_(pendingTaskCountMax), + firstTime_(true) { + } + + void start() { + ThreadManager::Impl::pendingTaskCountMax(pendingTaskCountMax_); + ThreadManager::Impl::start(); + addWorker(workerCount_); + } + + private: + const size_t workerCount_; + const size_t pendingTaskCountMax_; + bool firstTime_; + Monitor monitor_; +}; + + +shared_ptr ThreadManager::newThreadManager() { + return shared_ptr(new ThreadManager::Impl()); +} + +shared_ptr ThreadManager::newSimpleThreadManager(size_t count, size_t pendingTaskCountMax) { + return shared_ptr(new SimpleThreadManager(count, pendingTaskCountMax)); +} + +}}} // apache::thrift::concurrency + diff --git a/lib/cpp/src/concurrency/ThreadManager.h b/lib/cpp/src/concurrency/ThreadManager.h new file mode 100644 index 00000000..6e5a1781 --- /dev/null +++ b/lib/cpp/src/concurrency/ThreadManager.h @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_THREADMANAGER_H_ +#define _THRIFT_CONCURRENCY_THREADMANAGER_H_ 1 + +#include +#include +#include "Thread.h" + +namespace apache { namespace thrift { namespace concurrency { + +/** + * Thread Pool Manager and related classes + * + * @version $Id:$ + */ +class ThreadManager; + +/** + * ThreadManager class + * + * This class manages a pool of threads. It uses a ThreadFactory to create + * threads. It never actually creates or destroys worker threads, rather + * It maintains statistics on number of idle threads, number of active threads, + * task backlog, and average wait and service times and informs the PoolPolicy + * object bound to instances of this manager of interesting transitions. It is + * then up the PoolPolicy object to decide if the thread pool size needs to be + * adjusted and call this object addWorker and removeWorker methods to make + * changes. + * + * This design allows different policy implementations to used this code to + * handle basic worker thread management and worker task execution and focus on + * policy issues. The simplest policy, StaticPolicy, does nothing other than + * create a fixed number of threads. + */ +class ThreadManager { + + protected: + ThreadManager() {} + + public: + virtual ~ThreadManager() {} + + /** + * Starts the thread manager. Verifies all attributes have been properly + * initialized, then allocates necessary resources to begin operation + */ + virtual void start() = 0; + + /** + * Stops the thread manager. Aborts all remaining unprocessed task, shuts + * down all created worker threads, and realeases all allocated resources. + * This method blocks for all worker threads to complete, thus it can + * potentially block forever if a worker thread is running a task that + * won't terminate. + */ + virtual void stop() = 0; + + /** + * Joins the thread manager. This is the same as stop, except that it will + * block until all the workers have finished their work. At that point + * the ThreadManager will transition into the STOPPED state. + */ + virtual void join() = 0; + + enum STATE { + UNINITIALIZED, + STARTING, + STARTED, + JOINING, + STOPPING, + STOPPED + }; + + virtual const STATE state() const = 0; + + virtual boost::shared_ptr threadFactory() const = 0; + + virtual void threadFactory(boost::shared_ptr value) = 0; + + virtual void addWorker(size_t value=1) = 0; + + virtual void removeWorker(size_t value=1) = 0; + + /** + * Gets the current number of idle worker threads + */ + virtual size_t idleWorkerCount() const = 0; + + /** + * Gets the current number of total worker threads + */ + virtual size_t workerCount() const = 0; + + /** + * Gets the current number of pending tasks + */ + virtual size_t pendingTaskCount() const = 0; + + /** + * Gets the current number of pending and executing tasks + */ + virtual size_t totalTaskCount() const = 0; + + /** + * Gets the maximum pending task count. 0 indicates no maximum + */ + virtual size_t pendingTaskCountMax() const = 0; + + /** + * Adds a task to be executed at some time in the future by a worker thread. + * + * This method will block if pendingTaskCountMax() in not zero and pendingTaskCount() + * is greater than or equalt to pendingTaskCountMax(). If this method is called in the + * context of a ThreadManager worker thread it will throw a + * TooManyPendingTasksException + * + * @param task The task to queue for execution + * + * @param timeout Time to wait in milliseconds to add a task when a pending-task-count + * is specified. Specific cases: + * timeout = 0 : Wait forever to queue task. + * timeout = -1 : Return immediately if pending task count exceeds specified max + * + * @throws TooManyPendingTasksException Pending task count exceeds max pending task count + */ + virtual void add(boost::shared_ptrtask, int64_t timeout=0LL) = 0; + + /** + * Removes a pending task + */ + virtual void remove(boost::shared_ptr task) = 0; + + static boost::shared_ptr newThreadManager(); + + /** + * Creates a simple thread manager the uses count number of worker threads and has + * a pendingTaskCountMax maximum pending tasks. The default, 0, specified no limit + * on pending tasks + */ + static boost::shared_ptr newSimpleThreadManager(size_t count=4, size_t pendingTaskCountMax=0); + + class Task; + + class Worker; + + class Impl; +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_THREADMANAGER_H_ diff --git a/lib/cpp/src/concurrency/TimerManager.cpp b/lib/cpp/src/concurrency/TimerManager.cpp new file mode 100644 index 00000000..25515dc8 --- /dev/null +++ b/lib/cpp/src/concurrency/TimerManager.cpp @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "TimerManager.h" +#include "Exception.h" +#include "Util.h" + +#include +#include +#include + +namespace apache { namespace thrift { namespace concurrency { + +using boost::shared_ptr; + +typedef std::multimap >::iterator task_iterator; +typedef std::pair task_range; + +/** + * TimerManager class + * + * @version $Id:$ + */ +class TimerManager::Task : public Runnable { + + public: + enum STATE { + WAITING, + EXECUTING, + CANCELLED, + COMPLETE + }; + + Task(shared_ptr runnable) : + runnable_(runnable), + state_(WAITING) {} + + ~Task() { + } + + void run() { + if (state_ == EXECUTING) { + runnable_->run(); + state_ = COMPLETE; + } + } + + private: + shared_ptr runnable_; + class TimerManager::Dispatcher; + friend class TimerManager::Dispatcher; + STATE state_; +}; + +class TimerManager::Dispatcher: public Runnable { + + public: + Dispatcher(TimerManager* manager) : + manager_(manager) {} + + ~Dispatcher() {} + + /** + * Dispatcher entry point + * + * As long as dispatcher thread is running, pull tasks off the task taskMap_ + * and execute. + */ + void run() { + { + Synchronized s(manager_->monitor_); + if (manager_->state_ == TimerManager::STARTING) { + manager_->state_ = TimerManager::STARTED; + manager_->monitor_.notifyAll(); + } + } + + do { + std::set > expiredTasks; + { + Synchronized s(manager_->monitor_); + task_iterator expiredTaskEnd; + int64_t now = Util::currentTime(); + while (manager_->state_ == TimerManager::STARTED && + (expiredTaskEnd = manager_->taskMap_.upper_bound(now)) == manager_->taskMap_.begin()) { + int64_t timeout = 0LL; + if (!manager_->taskMap_.empty()) { + timeout = manager_->taskMap_.begin()->first - now; + } + assert((timeout != 0 && manager_->taskCount_ > 0) || (timeout == 0 && manager_->taskCount_ == 0)); + try { + manager_->monitor_.wait(timeout); + } catch (TimedOutException &e) {} + now = Util::currentTime(); + } + + if (manager_->state_ == TimerManager::STARTED) { + for (task_iterator ix = manager_->taskMap_.begin(); ix != expiredTaskEnd; ix++) { + shared_ptr task = ix->second; + expiredTasks.insert(task); + if (task->state_ == TimerManager::Task::WAITING) { + task->state_ = TimerManager::Task::EXECUTING; + } + manager_->taskCount_--; + } + manager_->taskMap_.erase(manager_->taskMap_.begin(), expiredTaskEnd); + } + } + + for (std::set >::iterator ix = expiredTasks.begin(); ix != expiredTasks.end(); ix++) { + (*ix)->run(); + } + + } while (manager_->state_ == TimerManager::STARTED); + + { + Synchronized s(manager_->monitor_); + if (manager_->state_ == TimerManager::STOPPING) { + manager_->state_ = TimerManager::STOPPED; + manager_->monitor_.notify(); + } + } + return; + } + + private: + TimerManager* manager_; + friend class TimerManager; +}; + +TimerManager::TimerManager() : + taskCount_(0), + state_(TimerManager::UNINITIALIZED), + dispatcher_(shared_ptr(new Dispatcher(this))) { +} + + +TimerManager::~TimerManager() { + + // If we haven't been explicitly stopped, do so now. We don't need to grab + // the monitor here, since stop already takes care of reentrancy. + + if (state_ != STOPPED) { + try { + stop(); + } catch(...) { + throw; + // uhoh + } + } +} + +void TimerManager::start() { + bool doStart = false; + { + Synchronized s(monitor_); + if (threadFactory_ == NULL) { + throw InvalidArgumentException(); + } + if (state_ == TimerManager::UNINITIALIZED) { + state_ = TimerManager::STARTING; + doStart = true; + } + } + + if (doStart) { + dispatcherThread_ = threadFactory_->newThread(dispatcher_); + dispatcherThread_->start(); + } + + { + Synchronized s(monitor_); + while (state_ == TimerManager::STARTING) { + monitor_.wait(); + } + assert(state_ != TimerManager::STARTING); + } +} + +void TimerManager::stop() { + bool doStop = false; + { + Synchronized s(monitor_); + if (state_ == TimerManager::UNINITIALIZED) { + state_ = TimerManager::STOPPED; + } else if (state_ != STOPPING && state_ != STOPPED) { + doStop = true; + state_ = STOPPING; + monitor_.notifyAll(); + } + while (state_ != STOPPED) { + monitor_.wait(); + } + } + + if (doStop) { + // Clean up any outstanding tasks + for (task_iterator ix = taskMap_.begin(); ix != taskMap_.end(); ix++) { + taskMap_.erase(ix); + } + + // Remove dispatcher's reference to us. + dispatcher_->manager_ = NULL; + } +} + +shared_ptr TimerManager::threadFactory() const { + Synchronized s(monitor_); + return threadFactory_; +} + +void TimerManager::threadFactory(shared_ptr value) { + Synchronized s(monitor_); + threadFactory_ = value; +} + +size_t TimerManager::taskCount() const { + return taskCount_; +} + +void TimerManager::add(shared_ptr task, int64_t timeout) { + int64_t now = Util::currentTime(); + timeout += now; + + { + Synchronized s(monitor_); + if (state_ != TimerManager::STARTED) { + throw IllegalStateException(); + } + + taskCount_++; + taskMap_.insert(std::pair >(timeout, shared_ptr(new Task(task)))); + + // If the task map was empty, or if we have an expiration that is earlier + // than any previously seen, kick the dispatcher so it can update its + // timeout + if (taskCount_ == 1 || timeout < taskMap_.begin()->first) { + monitor_.notify(); + } + } +} + +void TimerManager::add(shared_ptr task, const struct timespec& value) { + + int64_t expiration; + Util::toMilliseconds(expiration, value); + + int64_t now = Util::currentTime(); + + if (expiration < now) { + throw InvalidArgumentException(); + } + + add(task, expiration - now); +} + + +void TimerManager::remove(shared_ptr task) { + Synchronized s(monitor_); + if (state_ != TimerManager::STARTED) { + throw IllegalStateException(); + } +} + +const TimerManager::STATE TimerManager::state() const { return state_; } + +}}} // apache::thrift::concurrency + diff --git a/lib/cpp/src/concurrency/TimerManager.h b/lib/cpp/src/concurrency/TimerManager.h new file mode 100644 index 00000000..f3f799f9 --- /dev/null +++ b/lib/cpp/src/concurrency/TimerManager.h @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_TIMERMANAGER_H_ +#define _THRIFT_CONCURRENCY_TIMERMANAGER_H_ 1 + +#include "Exception.h" +#include "Monitor.h" +#include "Thread.h" + +#include +#include +#include + +namespace apache { namespace thrift { namespace concurrency { + +/** + * Timer Manager + * + * This class dispatches timer tasks when they fall due. + * + * @version $Id:$ + */ +class TimerManager { + + public: + + TimerManager(); + + virtual ~TimerManager(); + + virtual boost::shared_ptr threadFactory() const; + + virtual void threadFactory(boost::shared_ptr value); + + /** + * Starts the timer manager service + * + * @throws IllegalArgumentException Missing thread factory attribute + */ + virtual void start(); + + /** + * Stops the timer manager service + */ + virtual void stop(); + + virtual size_t taskCount() const ; + + /** + * Adds a task to be executed at some time in the future by a worker thread. + * + * @param task The task to execute + * @param timeout Time in milliseconds to delay before executing task + */ + virtual void add(boost::shared_ptr task, int64_t timeout); + + /** + * Adds a task to be executed at some time in the future by a worker thread. + * + * @param task The task to execute + * @param timeout Absolute time in the future to execute task. + */ + virtual void add(boost::shared_ptr task, const struct timespec& timeout); + + /** + * Removes a pending task + * + * @throws NoSuchTaskException Specified task doesn't exist. It was either + * processed already or this call was made for a + * task that was never added to this timer + * + * @throws UncancellableTaskException Specified task is already being + * executed or has completed execution. + */ + virtual void remove(boost::shared_ptr task); + + enum STATE { + UNINITIALIZED, + STARTING, + STARTED, + STOPPING, + STOPPED + }; + + virtual const STATE state() const; + + private: + boost::shared_ptr threadFactory_; + class Task; + friend class Task; + std::multimap > taskMap_; + size_t taskCount_; + Monitor monitor_; + STATE state_; + class Dispatcher; + friend class Dispatcher; + boost::shared_ptr dispatcher_; + boost::shared_ptr dispatcherThread_; +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_TIMERMANAGER_H_ diff --git a/lib/cpp/src/concurrency/Util.cpp b/lib/cpp/src/concurrency/Util.cpp new file mode 100644 index 00000000..1c449371 --- /dev/null +++ b/lib/cpp/src/concurrency/Util.cpp @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "Util.h" + +#ifdef HAVE_CONFIG_H +#include +#endif + +#if defined(HAVE_CLOCK_GETTIME) +#include +#elif defined(HAVE_GETTIMEOFDAY) +#include +#endif // defined(HAVE_CLOCK_GETTIME) + +namespace apache { namespace thrift { namespace concurrency { + +const int64_t Util::currentTime() { + int64_t result; + +#if defined(HAVE_CLOCK_GETTIME) + struct timespec now; + int ret = clock_gettime(CLOCK_REALTIME, &now); + assert(ret == 0); + toMilliseconds(result, now); +#elif defined(HAVE_GETTIMEOFDAY) + struct timeval now; + int ret = gettimeofday(&now, NULL); + assert(ret == 0); + toMilliseconds(result, now); +#else +#error "No high-precision clock is available." +#endif // defined(HAVE_CLOCK_GETTIME) + + return result; +} + + +}}} // apache::thrift::concurrency diff --git a/lib/cpp/src/concurrency/Util.h b/lib/cpp/src/concurrency/Util.h new file mode 100644 index 00000000..25fcc208 --- /dev/null +++ b/lib/cpp/src/concurrency/Util.h @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_CONCURRENCY_UTIL_H_ +#define _THRIFT_CONCURRENCY_UTIL_H_ 1 + +#include +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace concurrency { + +/** + * Utility methods + * + * This class contains basic utility methods for converting time formats, + * and other common platform-dependent concurrency operations. + * It should not be included in API headers for other concurrency library + * headers, since it will, by definition, pull in all sorts of horrid + * platform dependent crap. Rather it should be inluded directly in + * concurrency library implementation source. + * + * @version $Id:$ + */ +class Util { + + static const int64_t NS_PER_S = 1000000000LL; + static const int64_t US_PER_S = 1000000LL; + static const int64_t MS_PER_S = 1000LL; + + static const int64_t NS_PER_MS = NS_PER_S / MS_PER_S; + static const int64_t US_PER_MS = US_PER_S / MS_PER_S; + + public: + + /** + * Converts millisecond timestamp into a timespec struct + * + * @param struct timespec& result + * @param time or duration in milliseconds + */ + static void toTimespec(struct timespec& result, int64_t value) { + result.tv_sec = value / MS_PER_S; // ms to s + result.tv_nsec = (value % MS_PER_S) * NS_PER_MS; // ms to ns + } + + static void toTimeval(struct timeval& result, int64_t value) { + result.tv_sec = value / MS_PER_S; // ms to s + result.tv_usec = (value % MS_PER_S) * US_PER_MS; // ms to us + } + + /** + * Converts struct timespec to milliseconds + */ + static const void toMilliseconds(int64_t& result, const struct timespec& value) { + result = (value.tv_sec * MS_PER_S) + (value.tv_nsec / NS_PER_MS); + // round up -- int64_t cast is to avoid a compiler error for some GCCs + if (int64_t(value.tv_nsec) % NS_PER_MS >= (NS_PER_MS / 2)) { + ++result; + } + } + + /** + * Converts struct timeval to milliseconds + */ + static const void toMilliseconds(int64_t& result, const struct timeval& value) { + result = (value.tv_sec * MS_PER_S) + (value.tv_usec / US_PER_MS); + // round up -- int64_t cast is to avoid a compiler error for some GCCs + if (int64_t(value.tv_usec) % US_PER_MS >= (US_PER_MS / 2)) { + ++result; + } + } + + /** + * Get current time as milliseconds from epoch + */ + static const int64_t currentTime(); +}; + +}}} // apache::thrift::concurrency + +#endif // #ifndef _THRIFT_CONCURRENCY_UTIL_H_ diff --git a/lib/cpp/src/concurrency/test/Tests.cpp b/lib/cpp/src/concurrency/test/Tests.cpp new file mode 100644 index 00000000..c80bb883 --- /dev/null +++ b/lib/cpp/src/concurrency/test/Tests.cpp @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include "ThreadFactoryTests.h" +#include "TimerManagerTests.h" +#include "ThreadManagerTests.h" + +int main(int argc, char** argv) { + + std::string arg; + + std::vector args(argc - 1 > 1 ? argc - 1 : 1); + + args[0] = "all"; + + for (int ix = 1; ix < argc; ix++) { + args[ix - 1] = std::string(argv[ix]); + } + + bool runAll = args[0].compare("all") == 0; + + if (runAll || args[0].compare("thread-factory") == 0) { + + ThreadFactoryTests threadFactoryTests; + + std::cout << "ThreadFactory tests..." << std::endl; + + size_t count = 1000; + size_t floodLoops = 1; + size_t floodCount = 100000; + + std::cout << "\t\tThreadFactory reap N threads test: N = " << count << std::endl; + + assert(threadFactoryTests.reapNThreads(count)); + + std::cout << "\t\tThreadFactory floodN threads test: N = " << floodCount << std::endl; + + assert(threadFactoryTests.floodNTest(floodLoops, floodCount)); + + std::cout << "\t\tThreadFactory synchronous start test" << std::endl; + + assert(threadFactoryTests.synchStartTest()); + + std::cout << "\t\tThreadFactory monitor timeout test" << std::endl; + + assert(threadFactoryTests.monitorTimeoutTest()); + } + + if (runAll || args[0].compare("util") == 0) { + + std::cout << "Util tests..." << std::endl; + + std::cout << "\t\tUtil minimum time" << std::endl; + + int64_t time00 = Util::currentTime(); + int64_t time01 = Util::currentTime(); + + std::cout << "\t\t\tMinimum time: " << time01 - time00 << "ms" << std::endl; + + time00 = Util::currentTime(); + time01 = time00; + size_t count = 0; + + while (time01 < time00 + 10) { + count++; + time01 = Util::currentTime(); + } + + std::cout << "\t\t\tscall per ms: " << count / (time01 - time00) << std::endl; + } + + + if (runAll || args[0].compare("timer-manager") == 0) { + + std::cout << "TimerManager tests..." << std::endl; + + std::cout << "\t\tTimerManager test00" << std::endl; + + TimerManagerTests timerManagerTests; + + assert(timerManagerTests.test00()); + } + + if (runAll || args[0].compare("thread-manager") == 0) { + + std::cout << "ThreadManager tests..." << std::endl; + + { + + size_t workerCount = 100; + + size_t taskCount = 100000; + + int64_t delay = 10LL; + + std::cout << "\t\tThreadManager load test: worker count: " << workerCount << " task count: " << taskCount << " delay: " << delay << std::endl; + + ThreadManagerTests threadManagerTests; + + assert(threadManagerTests.loadTest(taskCount, delay, workerCount)); + + std::cout << "\t\tThreadManager block test: worker count: " << workerCount << " delay: " << delay << std::endl; + + assert(threadManagerTests.blockTest(delay, workerCount)); + + } + } + + if (runAll || args[0].compare("thread-manager-benchmark") == 0) { + + std::cout << "ThreadManager benchmark tests..." << std::endl; + + { + + size_t minWorkerCount = 2; + + size_t maxWorkerCount = 512; + + size_t tasksPerWorker = 1000; + + int64_t delay = 10LL; + + for (size_t workerCount = minWorkerCount; workerCount < maxWorkerCount; workerCount*= 2) { + + size_t taskCount = workerCount * tasksPerWorker; + + std::cout << "\t\tThreadManager load test: worker count: " << workerCount << " task count: " << taskCount << " delay: " << delay << std::endl; + + ThreadManagerTests threadManagerTests; + + threadManagerTests.loadTest(taskCount, delay, workerCount); + } + } + } +} diff --git a/lib/cpp/src/concurrency/test/ThreadFactoryTests.h b/lib/cpp/src/concurrency/test/ThreadFactoryTests.h new file mode 100644 index 00000000..859fbaf5 --- /dev/null +++ b/lib/cpp/src/concurrency/test/ThreadFactoryTests.h @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace apache { namespace thrift { namespace concurrency { namespace test { + +using boost::shared_ptr; +using namespace apache::thrift::concurrency; + +/** + * ThreadManagerTests class + * + * @version $Id:$ + */ +class ThreadFactoryTests { + +public: + + static const double ERROR; + + class Task: public Runnable { + + public: + + Task() {} + + void run() { + std::cout << "\t\t\tHello World" << std::endl; + } + }; + + /** + * Hello world test + */ + bool helloWorldTest() { + + PosixThreadFactory threadFactory = PosixThreadFactory(); + + shared_ptr task = shared_ptr(new ThreadFactoryTests::Task()); + + shared_ptr thread = threadFactory.newThread(task); + + thread->start(); + + thread->join(); + + std::cout << "\t\t\tSuccess!" << std::endl; + + return true; + } + + /** + * Reap N threads + */ + class ReapNTask: public Runnable { + + public: + + ReapNTask(Monitor& monitor, int& activeCount) : + _monitor(monitor), + _count(activeCount) {} + + void run() { + Synchronized s(_monitor); + + _count--; + + //std::cout << "\t\t\tthread count: " << _count << std::endl; + + if (_count == 0) { + _monitor.notify(); + } + } + + Monitor& _monitor; + + int& _count; + }; + + bool reapNThreads(int loop=1, int count=10) { + + PosixThreadFactory threadFactory = PosixThreadFactory(); + + Monitor* monitor = new Monitor(); + + for(int lix = 0; lix < loop; lix++) { + + int* activeCount = new int(count); + + std::set > threads; + + int tix; + + for (tix = 0; tix < count; tix++) { + try { + threads.insert(threadFactory.newThread(shared_ptr(new ReapNTask(*monitor, *activeCount)))); + } catch(SystemResourceException& e) { + std::cout << "\t\t\tfailed to create " << lix * count + tix << " thread " << e.what() << std::endl; + throw e; + } + } + + tix = 0; + for (std::set >::const_iterator thread = threads.begin(); thread != threads.end(); tix++, ++thread) { + + try { + (*thread)->start(); + } catch(SystemResourceException& e) { + std::cout << "\t\t\tfailed to start " << lix * count + tix << " thread " << e.what() << std::endl; + throw e; + } + } + + { + Synchronized s(*monitor); + while (*activeCount > 0) { + monitor->wait(1000); + } + } + + for (std::set >::const_iterator thread = threads.begin(); thread != threads.end(); thread++) { + threads.erase(*thread); + } + + std::cout << "\t\t\treaped " << lix * count << " threads" << std::endl; + } + + std::cout << "\t\t\tSuccess!" << std::endl; + + return true; + } + + class SynchStartTask: public Runnable { + + public: + + enum STATE { + UNINITIALIZED, + STARTING, + STARTED, + STOPPING, + STOPPED + }; + + SynchStartTask(Monitor& monitor, volatile STATE& state) : + _monitor(monitor), + _state(state) {} + + void run() { + { + Synchronized s(_monitor); + if (_state == SynchStartTask::STARTING) { + _state = SynchStartTask::STARTED; + _monitor.notify(); + } + } + + { + Synchronized s(_monitor); + while (_state == SynchStartTask::STARTED) { + _monitor.wait(); + } + + if (_state == SynchStartTask::STOPPING) { + _state = SynchStartTask::STOPPED; + _monitor.notifyAll(); + } + } + } + + private: + Monitor& _monitor; + volatile STATE& _state; + }; + + bool synchStartTest() { + + Monitor monitor; + + SynchStartTask::STATE state = SynchStartTask::UNINITIALIZED; + + shared_ptr task = shared_ptr(new SynchStartTask(monitor, state)); + + PosixThreadFactory threadFactory = PosixThreadFactory(); + + shared_ptr thread = threadFactory.newThread(task); + + if (state == SynchStartTask::UNINITIALIZED) { + + state = SynchStartTask::STARTING; + + thread->start(); + } + + { + Synchronized s(monitor); + while (state == SynchStartTask::STARTING) { + monitor.wait(); + } + } + + assert(state != SynchStartTask::STARTING); + + { + Synchronized s(monitor); + + try { + monitor.wait(100); + } catch(TimedOutException& e) { + } + + if (state == SynchStartTask::STARTED) { + + state = SynchStartTask::STOPPING; + + monitor.notify(); + } + + while (state == SynchStartTask::STOPPING) { + monitor.wait(); + } + } + + assert(state == SynchStartTask::STOPPED); + + bool success = true; + + std::cout << "\t\t\t" << (success ? "Success" : "Failure") << "!" << std::endl; + + return true; + } + + /** See how accurate monitor timeout is. */ + + bool monitorTimeoutTest(size_t count=1000, int64_t timeout=10) { + + Monitor monitor; + + int64_t startTime = Util::currentTime(); + + for (size_t ix = 0; ix < count; ix++) { + { + Synchronized s(monitor); + try { + monitor.wait(timeout); + } catch(TimedOutException& e) { + } + } + } + + int64_t endTime = Util::currentTime(); + + double error = ((endTime - startTime) - (count * timeout)) / (double)(count * timeout); + + if (error < 0.0) { + + error *= 1.0; + } + + bool success = error < ThreadFactoryTests::ERROR; + + std::cout << "\t\t\t" << (success ? "Success" : "Failure") << "! expected time: " << count * timeout << "ms elapsed time: "<< endTime - startTime << "ms error%: " << error * 100.0 << std::endl; + + return success; + } + + + class FloodTask : public Runnable { + public: + + FloodTask(const size_t id) :_id(id) {} + ~FloodTask(){ + if(_id % 1000 == 0) { + std::cout << "\t\tthread " << _id << " done" << std::endl; + } + } + + void run(){ + if(_id % 1000 == 0) { + std::cout << "\t\tthread " << _id << " started" << std::endl; + } + + usleep(1); + } + const size_t _id; + }; + + void foo(PosixThreadFactory *tf) { + } + + bool floodNTest(size_t loop=1, size_t count=100000) { + + bool success = false; + + for(size_t lix = 0; lix < loop; lix++) { + + PosixThreadFactory threadFactory = PosixThreadFactory(); + threadFactory.setDetached(true); + + for(size_t tix = 0; tix < count; tix++) { + + try { + + shared_ptr task(new FloodTask(lix * count + tix )); + + shared_ptr thread = threadFactory.newThread(task); + + thread->start(); + + usleep(1); + + } catch (TException& e) { + + std::cout << "\t\t\tfailed to start " << lix * count + tix << " thread " << e.what() << std::endl; + + return success; + } + } + + std::cout << "\t\t\tflooded " << (lix + 1) * count << " threads" << std::endl; + + success = true; + } + + return success; + } +}; + +const double ThreadFactoryTests::ERROR = .20; + +}}}} // apache::thrift::concurrency::test + diff --git a/lib/cpp/src/concurrency/test/ThreadManagerTests.h b/lib/cpp/src/concurrency/test/ThreadManagerTests.h new file mode 100644 index 00000000..e7b51743 --- /dev/null +++ b/lib/cpp/src/concurrency/test/ThreadManagerTests.h @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace concurrency { namespace test { + +using namespace apache::thrift::concurrency; + +/** + * ThreadManagerTests class + * + * @version $Id:$ + */ +class ThreadManagerTests { + +public: + + static const double ERROR; + + class Task: public Runnable { + + public: + + Task(Monitor& monitor, size_t& count, int64_t timeout) : + _monitor(monitor), + _count(count), + _timeout(timeout), + _done(false) {} + + void run() { + + _startTime = Util::currentTime(); + + { + Synchronized s(_sleep); + + try { + _sleep.wait(_timeout); + } catch(TimedOutException& e) { + ; + }catch(...) { + assert(0); + } + } + + _endTime = Util::currentTime(); + + _done = true; + + { + Synchronized s(_monitor); + + // std::cout << "Thread " << _count << " completed " << std::endl; + + _count--; + + if (_count == 0) { + + _monitor.notify(); + } + } + } + + Monitor& _monitor; + size_t& _count; + int64_t _timeout; + int64_t _startTime; + int64_t _endTime; + bool _done; + Monitor _sleep; + }; + + /** + * Dispatch count tasks, each of which blocks for timeout milliseconds then + * completes. Verify that all tasks completed and that thread manager cleans + * up properly on delete. + */ + bool loadTest(size_t count=100, int64_t timeout=100LL, size_t workerCount=4) { + + Monitor monitor; + + size_t activeCount = count; + + shared_ptr threadManager = ThreadManager::newSimpleThreadManager(workerCount); + + shared_ptr threadFactory = shared_ptr(new PosixThreadFactory()); + + threadFactory->setPriority(PosixThreadFactory::HIGHEST); + + threadManager->threadFactory(threadFactory); + + threadManager->start(); + + std::set > tasks; + + for (size_t ix = 0; ix < count; ix++) { + + tasks.insert(shared_ptr(new ThreadManagerTests::Task(monitor, activeCount, timeout))); + } + + int64_t time00 = Util::currentTime(); + + for (std::set >::iterator ix = tasks.begin(); ix != tasks.end(); ix++) { + + threadManager->add(*ix); + } + + { + Synchronized s(monitor); + + while(activeCount > 0) { + + monitor.wait(); + } + } + + int64_t time01 = Util::currentTime(); + + int64_t firstTime = 9223372036854775807LL; + int64_t lastTime = 0; + + double averageTime = 0; + int64_t minTime = 9223372036854775807LL; + int64_t maxTime = 0; + + for (std::set >::iterator ix = tasks.begin(); ix != tasks.end(); ix++) { + + shared_ptr task = *ix; + + int64_t delta = task->_endTime - task->_startTime; + + assert(delta > 0); + + if (task->_startTime < firstTime) { + firstTime = task->_startTime; + } + + if (task->_endTime > lastTime) { + lastTime = task->_endTime; + } + + if (delta < minTime) { + minTime = delta; + } + + if (delta > maxTime) { + maxTime = delta; + } + + averageTime+= delta; + } + + averageTime /= count; + + std::cout << "\t\t\tfirst start: " << firstTime << "ms Last end: " << lastTime << "ms min: " << minTime << "ms max: " << maxTime << "ms average: " << averageTime << "ms" << std::endl; + + double expectedTime = ((count + (workerCount - 1)) / workerCount) * timeout; + + double error = ((time01 - time00) - expectedTime) / expectedTime; + + if (error < 0) { + error*= -1.0; + } + + bool success = error < ERROR; + + std::cout << "\t\t\t" << (success ? "Success" : "Failure") << "! expected time: " << expectedTime << "ms elapsed time: "<< time01 - time00 << "ms error%: " << error * 100.0 << std::endl; + + return success; + } + + class BlockTask: public Runnable { + + public: + + BlockTask(Monitor& monitor, Monitor& bmonitor, size_t& count) : + _monitor(monitor), + _bmonitor(bmonitor), + _count(count) {} + + void run() { + { + Synchronized s(_bmonitor); + + _bmonitor.wait(); + + } + + { + Synchronized s(_monitor); + + _count--; + + if (_count == 0) { + + _monitor.notify(); + } + } + } + + Monitor& _monitor; + Monitor& _bmonitor; + size_t& _count; + }; + + /** + * Block test. Create pendingTaskCountMax tasks. Verify that we block adding the + * pendingTaskCountMax + 1th task. Verify that we unblock when a task completes */ + + bool blockTest(int64_t timeout=100LL, size_t workerCount=2) { + + bool success = false; + + try { + + Monitor bmonitor; + Monitor monitor; + + size_t pendingTaskMaxCount = workerCount; + + size_t activeCounts[] = {workerCount, pendingTaskMaxCount, 1}; + + shared_ptr threadManager = ThreadManager::newSimpleThreadManager(workerCount, pendingTaskMaxCount); + + shared_ptr threadFactory = shared_ptr(new PosixThreadFactory()); + + threadFactory->setPriority(PosixThreadFactory::HIGHEST); + + threadManager->threadFactory(threadFactory); + + threadManager->start(); + + std::set > tasks; + + for (size_t ix = 0; ix < workerCount; ix++) { + + tasks.insert(shared_ptr(new ThreadManagerTests::BlockTask(monitor, bmonitor,activeCounts[0]))); + } + + for (size_t ix = 0; ix < pendingTaskMaxCount; ix++) { + + tasks.insert(shared_ptr(new ThreadManagerTests::BlockTask(monitor, bmonitor,activeCounts[1]))); + } + + for (std::set >::iterator ix = tasks.begin(); ix != tasks.end(); ix++) { + threadManager->add(*ix); + } + + if(!(success = (threadManager->totalTaskCount() == pendingTaskMaxCount + workerCount))) { + throw TException("Unexpected pending task count"); + } + + shared_ptr extraTask(new ThreadManagerTests::BlockTask(monitor, bmonitor, activeCounts[2])); + + try { + threadManager->add(extraTask, 1); + throw TException("Unexpected success adding task in excess of pending task count"); + } catch(TimedOutException& e) { + } + + std::cout << "\t\t\t" << "Pending tasks " << threadManager->pendingTaskCount() << std::endl; + + { + Synchronized s(bmonitor); + + bmonitor.notifyAll(); + } + + { + Synchronized s(monitor); + + while(activeCounts[0] != 0) { + monitor.wait(); + } + } + + std::cout << "\t\t\t" << "Pending tasks " << threadManager->pendingTaskCount() << std::endl; + + try { + threadManager->add(extraTask, 1); + } catch(TimedOutException& e) { + std::cout << "\t\t\t" << "add timed out unexpectedly" << std::endl; + throw TException("Unexpected timeout adding task"); + + } catch(TooManyPendingTasksException& e) { + std::cout << "\t\t\t" << "add encountered too many pending exepctions" << std::endl; + throw TException("Unexpected timeout adding task"); + } + + // Wake up tasks that were pending before and wait for them to complete + + { + Synchronized s(bmonitor); + + bmonitor.notifyAll(); + } + + { + Synchronized s(monitor); + + while(activeCounts[1] != 0) { + monitor.wait(); + } + } + + // Wake up the extra task and wait for it to complete + + { + Synchronized s(bmonitor); + + bmonitor.notifyAll(); + } + + { + Synchronized s(monitor); + + while(activeCounts[2] != 0) { + monitor.wait(); + } + } + + if(!(success = (threadManager->totalTaskCount() == 0))) { + throw TException("Unexpected pending task count"); + } + + } catch(TException& e) { + } + + std::cout << "\t\t\t" << (success ? "Success" : "Failure") << std::endl; + return success; + } +}; + +const double ThreadManagerTests::ERROR = .20; + +}}}} // apache::thrift::concurrency + +using namespace apache::thrift::concurrency::test; + diff --git a/lib/cpp/src/concurrency/test/TimerManagerTests.h b/lib/cpp/src/concurrency/test/TimerManagerTests.h new file mode 100644 index 00000000..e6fe6ce7 --- /dev/null +++ b/lib/cpp/src/concurrency/test/TimerManagerTests.h @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include +#include + +namespace apache { namespace thrift { namespace concurrency { namespace test { + +using namespace apache::thrift::concurrency; + +/** + * ThreadManagerTests class + * + * @version $Id:$ + */ +class TimerManagerTests { + + public: + + static const double ERROR; + + class Task: public Runnable { + public: + + Task(Monitor& monitor, int64_t timeout) : + _timeout(timeout), + _startTime(Util::currentTime()), + _monitor(monitor), + _success(false), + _done(false) {} + + ~Task() { std::cerr << this << std::endl; } + + void run() { + + _endTime = Util::currentTime(); + + // Figure out error percentage + + int64_t delta = _endTime - _startTime; + + + delta = delta > _timeout ? delta - _timeout : _timeout - delta; + + float error = delta / _timeout; + + if(error < ERROR) { + _success = true; + } + + _done = true; + + std::cout << "\t\t\tTimerManagerTests::Task[" << this << "] done" << std::endl; //debug + + {Synchronized s(_monitor); + _monitor.notifyAll(); + } + } + + int64_t _timeout; + int64_t _startTime; + int64_t _endTime; + Monitor& _monitor; + bool _success; + bool _done; + }; + + /** + * This test creates two tasks and waits for the first to expire within 10% + * of the expected expiration time. It then verifies that the timer manager + * properly clean up itself and the remaining orphaned timeout task when the + * manager goes out of scope and its destructor is called. + */ + bool test00(int64_t timeout=1000LL) { + + shared_ptr orphanTask = shared_ptr(new TimerManagerTests::Task(_monitor, 10 * timeout)); + + { + + TimerManager timerManager; + + timerManager.threadFactory(shared_ptr(new PosixThreadFactory())); + + timerManager.start(); + + assert(timerManager.state() == TimerManager::STARTED); + + shared_ptr task = shared_ptr(new TimerManagerTests::Task(_monitor, timeout)); + + { + Synchronized s(_monitor); + + timerManager.add(orphanTask, 10 * timeout); + + timerManager.add(task, timeout); + + _monitor.wait(); + } + + assert(task->_done); + + + std::cout << "\t\t\t" << (task->_success ? "Success" : "Failure") << "!" << std::endl; + } + + // timerManager.stop(); This is where it happens via destructor + + assert(!orphanTask->_done); + + return true; + } + + friend class TestTask; + + Monitor _monitor; +}; + +const double TimerManagerTests::ERROR = .20; + +}}}} // apache::thrift::concurrency + diff --git a/lib/cpp/src/processor/PeekProcessor.cpp b/lib/cpp/src/processor/PeekProcessor.cpp new file mode 100644 index 00000000..c721861b --- /dev/null +++ b/lib/cpp/src/processor/PeekProcessor.cpp @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "PeekProcessor.h" + +using namespace apache::thrift::transport; +using namespace apache::thrift::protocol; +using namespace apache::thrift; + +namespace apache { namespace thrift { namespace processor { + +PeekProcessor::PeekProcessor() { + memoryBuffer_.reset(new TMemoryBuffer()); + targetTransport_ = memoryBuffer_; +} +PeekProcessor::~PeekProcessor() {} + +void PeekProcessor::initialize(boost::shared_ptr actualProcessor, + boost::shared_ptr protocolFactory, + boost::shared_ptr transportFactory) { + actualProcessor_ = actualProcessor; + pipedProtocol_ = protocolFactory->getProtocol(targetTransport_); + transportFactory_ = transportFactory; + transportFactory_->initializeTargetTransport(targetTransport_); +} + +boost::shared_ptr PeekProcessor::getPipedTransport(boost::shared_ptr in) { + return transportFactory_->getTransport(in); +} + +void PeekProcessor::setTargetTransport(boost::shared_ptr targetTransport) { + targetTransport_ = targetTransport; + if (boost::dynamic_pointer_cast(targetTransport_)) { + memoryBuffer_ = boost::dynamic_pointer_cast(targetTransport); + } else if (boost::dynamic_pointer_cast(targetTransport_)) { + memoryBuffer_ = boost::dynamic_pointer_cast(boost::dynamic_pointer_cast(targetTransport_)->getTargetTransport()); + } + + if (!memoryBuffer_) { + throw TException("Target transport must be a TMemoryBuffer or a TPipedTransport with TMemoryBuffer"); + } +} + +bool PeekProcessor::process(boost::shared_ptr in, + boost::shared_ptr out) { + + std::string fname; + TMessageType mtype; + int32_t seqid; + in->readMessageBegin(fname, mtype, seqid); + + if (mtype != T_CALL) { + throw TException("Unexpected message type"); + } + + // Peek at the name + peekName(fname); + + TType ftype; + int16_t fid; + while (true) { + in->readFieldBegin(fname, ftype, fid); + if (ftype == T_STOP) { + break; + } + + // Peek at the variable + peek(in, ftype, fid); + in->readFieldEnd(); + } + in->readMessageEnd(); + in->getTransport()->readEnd(); + + // + // All the data is now in memoryBuffer_ and ready to be processed + // + + // Let's first take a peek at the full data in memory + uint8_t* buffer; + uint32_t size; + memoryBuffer_->getBuffer(&buffer, &size); + peekBuffer(buffer, size); + + // Done peeking at variables + peekEnd(); + + bool ret = actualProcessor_->process(pipedProtocol_, out); + memoryBuffer_->resetBuffer(); + return ret; +} + +void PeekProcessor::peekName(const std::string& fname) { +} + +void PeekProcessor::peekBuffer(uint8_t* buffer, uint32_t size) { +} + +void PeekProcessor::peek(boost::shared_ptr in, + TType ftype, + int16_t fid) { + in->skip(ftype); +} + +void PeekProcessor::peekEnd() {} + +}}} diff --git a/lib/cpp/src/processor/PeekProcessor.h b/lib/cpp/src/processor/PeekProcessor.h new file mode 100644 index 00000000..0f7c016a --- /dev/null +++ b/lib/cpp/src/processor/PeekProcessor.h @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef PEEKPROCESSOR_H +#define PEEKPROCESSOR_H + +#include +#include +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace processor { + +/* + * Class for peeking at the raw data that is being processed by another processor + * and gives the derived class a chance to change behavior accordingly + * + */ +class PeekProcessor : public apache::thrift::TProcessor { + + public: + PeekProcessor(); + virtual ~PeekProcessor(); + + // Input here: actualProcessor - the underlying processor + // protocolFactory - the protocol factory used to wrap the memory buffer + // transportFactory - this TPipedTransportFactory is used to wrap the source transport + // via a call to getPipedTransport + void initialize(boost::shared_ptr actualProcessor, + boost::shared_ptr protocolFactory, + boost::shared_ptr transportFactory); + + boost::shared_ptr getPipedTransport(boost::shared_ptr in); + + void setTargetTransport(boost::shared_ptr targetTransport); + + virtual bool process(boost::shared_ptr in, + boost::shared_ptr out); + + // The following three functions can be overloaded by child classes to + // achieve desired peeking behavior + virtual void peekName(const std::string& fname); + virtual void peekBuffer(uint8_t* buffer, uint32_t size); + virtual void peek(boost::shared_ptr in, + apache::thrift::protocol::TType ftype, + int16_t fid); + virtual void peekEnd(); + + private: + boost::shared_ptr actualProcessor_; + boost::shared_ptr pipedProtocol_; + boost::shared_ptr transportFactory_; + boost::shared_ptr memoryBuffer_; + boost::shared_ptr targetTransport_; +}; + +}}} // apache::thrift::processor + +#endif diff --git a/lib/cpp/src/processor/StatsProcessor.h b/lib/cpp/src/processor/StatsProcessor.h new file mode 100644 index 00000000..820b3ad4 --- /dev/null +++ b/lib/cpp/src/processor/StatsProcessor.h @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef STATSPROCESSOR_H +#define STATSPROCESSOR_H + +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace processor { + +/* + * Class for keeping track of function call statistics and printing them if desired + * + */ +class StatsProcessor : public apache::thrift::TProcessor { +public: + StatsProcessor(bool print, bool frequency) + : print_(print), + frequency_(frequency) + {} + virtual ~StatsProcessor() {}; + + virtual bool process(boost::shared_ptr piprot, boost::shared_ptr poprot) { + + piprot_ = piprot; + + std::string fname; + apache::thrift::protocol::TMessageType mtype; + int32_t seqid; + + piprot_->readMessageBegin(fname, mtype, seqid); + if (mtype != apache::thrift::protocol::T_CALL) { + if (print_) { + printf("Unknown message type\n"); + } + throw apache::thrift::TException("Unexpected message type"); + } + if (print_) { + printf("%s (", fname.c_str()); + } + if (frequency_) { + if (frequency_map_.find(fname) != frequency_map_.end()) { + frequency_map_[fname]++; + } else { + frequency_map_[fname] = 1; + } + } + + apache::thrift::protocol::TType ftype; + int16_t fid; + + while (true) { + piprot_->readFieldBegin(fname, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + + printAndPassToBuffer(ftype); + if (print_) { + printf(", "); + } + } + + if (print_) { + printf("\b\b)\n"); + } + return true; + } + + const std::map& get_frequency_map() { + return frequency_map_; + } + +protected: + void printAndPassToBuffer(apache::thrift::protocol::TType ftype) { + switch (ftype) { + case apache::thrift::protocol::T_BOOL: + { + bool boolv; + piprot_->readBool(boolv); + if (print_) { + printf("%d", boolv); + } + } + break; + case apache::thrift::protocol::T_BYTE: + { + int8_t bytev; + piprot_->readByte(bytev); + if (print_) { + printf("%d", bytev); + } + } + break; + case apache::thrift::protocol::T_I16: + { + int16_t i16; + piprot_->readI16(i16); + if (print_) { + printf("%d", i16); + } + } + break; + case apache::thrift::protocol::T_I32: + { + int32_t i32; + piprot_->readI32(i32); + if (print_) { + printf("%d", i32); + } + } + break; + case apache::thrift::protocol::T_I64: + { + int64_t i64; + piprot_->readI64(i64); + if (print_) { + printf("%ld", i64); + } + } + break; + case apache::thrift::protocol::T_DOUBLE: + { + double dub; + piprot_->readDouble(dub); + if (print_) { + printf("%f", dub); + } + } + break; + case apache::thrift::protocol::T_STRING: + { + std::string str; + piprot_->readString(str); + if (print_) { + printf("%s", str.c_str()); + } + } + break; + case apache::thrift::protocol::T_STRUCT: + { + std::string name; + int16_t fid; + apache::thrift::protocol::TType ftype; + piprot_->readStructBegin(name); + if (print_) { + printf("<"); + } + while (true) { + piprot_->readFieldBegin(name, ftype, fid); + if (ftype == apache::thrift::protocol::T_STOP) { + break; + } + printAndPassToBuffer(ftype); + if (print_) { + printf(","); + } + piprot_->readFieldEnd(); + } + piprot_->readStructEnd(); + if (print_) { + printf("\b>"); + } + } + break; + case apache::thrift::protocol::T_MAP: + { + apache::thrift::protocol::TType keyType; + apache::thrift::protocol::TType valType; + uint32_t i, size; + piprot_->readMapBegin(keyType, valType, size); + if (print_) { + printf("{"); + } + for (i = 0; i < size; i++) { + printAndPassToBuffer(keyType); + if (print_) { + printf("=>"); + } + printAndPassToBuffer(valType); + if (print_) { + printf(","); + } + } + piprot_->readMapEnd(); + if (print_) { + printf("\b}"); + } + } + break; + case apache::thrift::protocol::T_SET: + { + apache::thrift::protocol::TType elemType; + uint32_t i, size; + piprot_->readSetBegin(elemType, size); + if (print_) { + printf("{"); + } + for (i = 0; i < size; i++) { + printAndPassToBuffer(elemType); + if (print_) { + printf(","); + } + } + piprot_->readSetEnd(); + if (print_) { + printf("\b}"); + } + } + break; + case apache::thrift::protocol::T_LIST: + { + apache::thrift::protocol::TType elemType; + uint32_t i, size; + piprot_->readListBegin(elemType, size); + if (print_) { + printf("["); + } + for (i = 0; i < size; i++) { + printAndPassToBuffer(elemType); + if (print_) { + printf(","); + } + } + piprot_->readListEnd(); + if (print_) { + printf("\b]"); + } + } + break; + default: + break; + } + } + + boost::shared_ptr piprot_; + std::map frequency_map_; + + bool print_; + bool frequency_; +}; + +}}} // apache::thrift::processor + +#endif diff --git a/lib/cpp/src/protocol/TBase64Utils.cpp b/lib/cpp/src/protocol/TBase64Utils.cpp new file mode 100644 index 00000000..14481c49 --- /dev/null +++ b/lib/cpp/src/protocol/TBase64Utils.cpp @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "TBase64Utils.h" + +#include + +using std::string; + +namespace apache { namespace thrift { namespace protocol { + + +static const uint8_t *kBase64EncodeTable = (const uint8_t *) + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +void base64_encode(const uint8_t *in, uint32_t len, uint8_t *buf) { + buf[0] = kBase64EncodeTable[(in[0] >> 2) & 0x3F]; + if (len == 3) { + buf[1] = kBase64EncodeTable[((in[0] << 4) + (in[1] >> 4)) & 0x3f]; + buf[2] = kBase64EncodeTable[((in[1] << 2) + (in[2] >> 6)) & 0x3f]; + buf[3] = kBase64EncodeTable[in[2] & 0x3f]; + } else if (len == 2) { + buf[1] = kBase64EncodeTable[((in[0] << 4) + (in[1] >> 4)) & 0x3f]; + buf[2] = kBase64EncodeTable[(in[1] << 2) & 0x3f]; + } else { // len == 1 + buf[1] = kBase64EncodeTable[(in[0] << 4) & 0x3f]; + } +} + +static const uint8_t kBase64DecodeTable[256] ={ + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,62,-1,-1,-1,63, + 52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-1,-1,-1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, + 15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1, + -1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, + 41,42,43,44,45,46,47,48,49,50,51,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +}; + +void base64_decode(uint8_t *buf, uint32_t len) { + buf[0] = (kBase64DecodeTable[buf[0]] << 2) | + (kBase64DecodeTable[buf[1]] >> 4); + if (len > 2) { + buf[1] = ((kBase64DecodeTable[buf[1]] << 4) & 0xf0) | + (kBase64DecodeTable[buf[2]] >> 2); + if (len > 3) { + buf[2] = ((kBase64DecodeTable[buf[2]] << 6) & 0xc0) | + (kBase64DecodeTable[buf[3]]); + } + } +} + + +}}} // apache::thrift::protocol diff --git a/lib/cpp/src/protocol/TBase64Utils.h b/lib/cpp/src/protocol/TBase64Utils.h new file mode 100644 index 00000000..3def7335 --- /dev/null +++ b/lib/cpp/src/protocol/TBase64Utils.h @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TBASE64UTILS_H_ +#define _THRIFT_PROTOCOL_TBASE64UTILS_H_ + +#include +#include + +namespace apache { namespace thrift { namespace protocol { + +// in must be at least len bytes +// len must be 1, 2, or 3 +// buf must be a buffer of at least 4 bytes and may not overlap in +// the data is not padded with '='; the caller can do this if desired +void base64_encode(const uint8_t *in, uint32_t len, uint8_t *buf); + +// buf must be a buffer of at least 4 bytes and contain base64 encoded values +// buf will be changed to contain output bytes +// len is number of bytes to consume from input (must be 2, 3, or 4) +// no '=' padding should be included in the input +void base64_decode(uint8_t *buf, uint32_t len); + +}}} // apache::thrift::protocol + +#endif // #define _THRIFT_PROTOCOL_TBASE64UTILS_H_ diff --git a/lib/cpp/src/protocol/TBinaryProtocol.cpp b/lib/cpp/src/protocol/TBinaryProtocol.cpp new file mode 100644 index 00000000..6a4838b4 --- /dev/null +++ b/lib/cpp/src/protocol/TBinaryProtocol.cpp @@ -0,0 +1,394 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "TBinaryProtocol.h" + +#include + +using std::string; + +namespace apache { namespace thrift { namespace protocol { + +uint32_t TBinaryProtocol::writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid) { + if (strict_write_) { + int32_t version = (VERSION_1) | ((int32_t)messageType); + uint32_t wsize = 0; + wsize += writeI32(version); + wsize += writeString(name); + wsize += writeI32(seqid); + return wsize; + } else { + uint32_t wsize = 0; + wsize += writeString(name); + wsize += writeByte((int8_t)messageType); + wsize += writeI32(seqid); + return wsize; + } +} + +uint32_t TBinaryProtocol::writeMessageEnd() { + return 0; +} + +uint32_t TBinaryProtocol::writeStructBegin(const char* name) { + return 0; +} + +uint32_t TBinaryProtocol::writeStructEnd() { + return 0; +} + +uint32_t TBinaryProtocol::writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId) { + uint32_t wsize = 0; + wsize += writeByte((int8_t)fieldType); + wsize += writeI16(fieldId); + return wsize; +} + +uint32_t TBinaryProtocol::writeFieldEnd() { + return 0; +} + +uint32_t TBinaryProtocol::writeFieldStop() { + return + writeByte((int8_t)T_STOP); +} + +uint32_t TBinaryProtocol::writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size) { + uint32_t wsize = 0; + wsize += writeByte((int8_t)keyType); + wsize += writeByte((int8_t)valType); + wsize += writeI32((int32_t)size); + return wsize; +} + +uint32_t TBinaryProtocol::writeMapEnd() { + return 0; +} + +uint32_t TBinaryProtocol::writeListBegin(const TType elemType, + const uint32_t size) { + uint32_t wsize = 0; + wsize += writeByte((int8_t) elemType); + wsize += writeI32((int32_t)size); + return wsize; +} + +uint32_t TBinaryProtocol::writeListEnd() { + return 0; +} + +uint32_t TBinaryProtocol::writeSetBegin(const TType elemType, + const uint32_t size) { + uint32_t wsize = 0; + wsize += writeByte((int8_t)elemType); + wsize += writeI32((int32_t)size); + return wsize; +} + +uint32_t TBinaryProtocol::writeSetEnd() { + return 0; +} + +uint32_t TBinaryProtocol::writeBool(const bool value) { + uint8_t tmp = value ? 1 : 0; + trans_->write(&tmp, 1); + return 1; +} + +uint32_t TBinaryProtocol::writeByte(const int8_t byte) { + trans_->write((uint8_t*)&byte, 1); + return 1; +} + +uint32_t TBinaryProtocol::writeI16(const int16_t i16) { + int16_t net = (int16_t)htons(i16); + trans_->write((uint8_t*)&net, 2); + return 2; +} + +uint32_t TBinaryProtocol::writeI32(const int32_t i32) { + int32_t net = (int32_t)htonl(i32); + trans_->write((uint8_t*)&net, 4); + return 4; +} + +uint32_t TBinaryProtocol::writeI64(const int64_t i64) { + int64_t net = (int64_t)htonll(i64); + trans_->write((uint8_t*)&net, 8); + return 8; +} + +uint32_t TBinaryProtocol::writeDouble(const double dub) { + BOOST_STATIC_ASSERT(sizeof(double) == sizeof(uint64_t)); + BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); + + uint64_t bits = bitwise_cast(dub); + bits = htonll(bits); + trans_->write((uint8_t*)&bits, 8); + return 8; +} + + +uint32_t TBinaryProtocol::writeString(const string& str) { + uint32_t size = str.size(); + uint32_t result = writeI32((int32_t)size); + if (size > 0) { + trans_->write((uint8_t*)str.data(), size); + } + return result + size; +} + +uint32_t TBinaryProtocol::writeBinary(const string& str) { + return TBinaryProtocol::writeString(str); +} + +/** + * Reading functions + */ + +uint32_t TBinaryProtocol::readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid) { + uint32_t result = 0; + int32_t sz; + result += readI32(sz); + + if (sz < 0) { + // Check for correct version number + int32_t version = sz & VERSION_MASK; + if (version != VERSION_1) { + throw TProtocolException(TProtocolException::BAD_VERSION, "Bad version identifier"); + } + messageType = (TMessageType)(sz & 0x000000ff); + result += readString(name); + result += readI32(seqid); + } else { + if (strict_read_) { + throw TProtocolException(TProtocolException::BAD_VERSION, "No version identifier... old protocol client in strict mode?"); + } else { + // Handle pre-versioned input + int8_t type; + result += readStringBody(name, sz); + result += readByte(type); + messageType = (TMessageType)type; + result += readI32(seqid); + } + } + return result; +} + +uint32_t TBinaryProtocol::readMessageEnd() { + return 0; +} + +uint32_t TBinaryProtocol::readStructBegin(string& name) { + name = ""; + return 0; +} + +uint32_t TBinaryProtocol::readStructEnd() { + return 0; +} + +uint32_t TBinaryProtocol::readFieldBegin(string& name, + TType& fieldType, + int16_t& fieldId) { + uint32_t result = 0; + int8_t type; + result += readByte(type); + fieldType = (TType)type; + if (fieldType == T_STOP) { + fieldId = 0; + return result; + } + result += readI16(fieldId); + return result; +} + +uint32_t TBinaryProtocol::readFieldEnd() { + return 0; +} + +uint32_t TBinaryProtocol::readMapBegin(TType& keyType, + TType& valType, + uint32_t& size) { + int8_t k, v; + uint32_t result = 0; + int32_t sizei; + result += readByte(k); + keyType = (TType)k; + result += readByte(v); + valType = (TType)v; + result += readI32(sizei); + if (sizei < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (container_limit_ && sizei > container_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + size = (uint32_t)sizei; + return result; +} + +uint32_t TBinaryProtocol::readMapEnd() { + return 0; +} + +uint32_t TBinaryProtocol::readListBegin(TType& elemType, + uint32_t& size) { + int8_t e; + uint32_t result = 0; + int32_t sizei; + result += readByte(e); + elemType = (TType)e; + result += readI32(sizei); + if (sizei < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (container_limit_ && sizei > container_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + size = (uint32_t)sizei; + return result; +} + +uint32_t TBinaryProtocol::readListEnd() { + return 0; +} + +uint32_t TBinaryProtocol::readSetBegin(TType& elemType, + uint32_t& size) { + int8_t e; + uint32_t result = 0; + int32_t sizei; + result += readByte(e); + elemType = (TType)e; + result += readI32(sizei); + if (sizei < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (container_limit_ && sizei > container_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + size = (uint32_t)sizei; + return result; +} + +uint32_t TBinaryProtocol::readSetEnd() { + return 0; +} + +uint32_t TBinaryProtocol::readBool(bool& value) { + uint8_t b[1]; + trans_->readAll(b, 1); + value = *(int8_t*)b != 0; + return 1; +} + +uint32_t TBinaryProtocol::readByte(int8_t& byte) { + uint8_t b[1]; + trans_->readAll(b, 1); + byte = *(int8_t*)b; + return 1; +} + +uint32_t TBinaryProtocol::readI16(int16_t& i16) { + uint8_t b[2]; + trans_->readAll(b, 2); + i16 = *(int16_t*)b; + i16 = (int16_t)ntohs(i16); + return 2; +} + +uint32_t TBinaryProtocol::readI32(int32_t& i32) { + uint8_t b[4]; + trans_->readAll(b, 4); + i32 = *(int32_t*)b; + i32 = (int32_t)ntohl(i32); + return 4; +} + +uint32_t TBinaryProtocol::readI64(int64_t& i64) { + uint8_t b[8]; + trans_->readAll(b, 8); + i64 = *(int64_t*)b; + i64 = (int64_t)ntohll(i64); + return 8; +} + +uint32_t TBinaryProtocol::readDouble(double& dub) { + BOOST_STATIC_ASSERT(sizeof(double) == sizeof(uint64_t)); + BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); + + uint64_t bits; + uint8_t b[8]; + trans_->readAll(b, 8); + bits = *(uint64_t*)b; + bits = ntohll(bits); + dub = bitwise_cast(bits); + return 8; +} + +uint32_t TBinaryProtocol::readString(string& str) { + uint32_t result; + int32_t size; + result = readI32(size); + return result + readStringBody(str, size); +} + +uint32_t TBinaryProtocol::readBinary(string& str) { + return TBinaryProtocol::readString(str); +} + +uint32_t TBinaryProtocol::readStringBody(string& str, int32_t size) { + uint32_t result = 0; + + // Catch error cases + if (size < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } + if (string_limit_ > 0 && size > string_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + + // Catch empty string case + if (size == 0) { + str = ""; + return result; + } + + // Use the heap here to prevent stack overflow for v. large strings + if (size > string_buf_size_ || string_buf_ == NULL) { + void* new_string_buf = std::realloc(string_buf_, (uint32_t)size); + if (new_string_buf == NULL) { + throw TProtocolException(TProtocolException::UNKNOWN, "Out of memory in TBinaryProtocol::readString"); + } + string_buf_ = (uint8_t*)new_string_buf; + string_buf_size_ = size; + } + trans_->readAll(string_buf_, size); + str = string((char*)string_buf_, size); + return (uint32_t)size; +} + +}}} // apache::thrift::protocol diff --git a/lib/cpp/src/protocol/TBinaryProtocol.h b/lib/cpp/src/protocol/TBinaryProtocol.h new file mode 100644 index 00000000..7fd3de67 --- /dev/null +++ b/lib/cpp/src/protocol/TBinaryProtocol.h @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TBINARYPROTOCOL_H_ +#define _THRIFT_PROTOCOL_TBINARYPROTOCOL_H_ 1 + +#include "TProtocol.h" + +#include + +namespace apache { namespace thrift { namespace protocol { + +/** + * The default binary protocol for thrift. Writes all data in a very basic + * binary format, essentially just spitting out the raw bytes. + * + */ +class TBinaryProtocol : public TProtocol { + protected: + static const int32_t VERSION_MASK = 0xffff0000; + static const int32_t VERSION_1 = 0x80010000; + // VERSION_2 (0x80020000) is taken by TDenseProtocol. + + public: + TBinaryProtocol(boost::shared_ptr trans) : + TProtocol(trans), + string_limit_(0), + container_limit_(0), + strict_read_(false), + strict_write_(true), + string_buf_(NULL), + string_buf_size_(0) {} + + TBinaryProtocol(boost::shared_ptr trans, + int32_t string_limit, + int32_t container_limit, + bool strict_read, + bool strict_write) : + TProtocol(trans), + string_limit_(string_limit), + container_limit_(container_limit), + strict_read_(strict_read), + strict_write_(strict_write), + string_buf_(NULL), + string_buf_size_(0) {} + + ~TBinaryProtocol() { + if (string_buf_ != NULL) { + std::free(string_buf_); + string_buf_size_ = 0; + } + } + + void setStringSizeLimit(int32_t string_limit) { + string_limit_ = string_limit; + } + + void setContainerSizeLimit(int32_t container_limit) { + container_limit_ = container_limit; + } + + void setStrict(bool strict_read, bool strict_write) { + strict_read_ = strict_read; + strict_write_ = strict_write; + } + + /** + * Writing functions. + */ + + virtual uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid); + + virtual uint32_t writeMessageEnd(); + + + uint32_t writeStructBegin(const char* name); + + uint32_t writeStructEnd(); + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId); + + uint32_t writeFieldEnd(); + + uint32_t writeFieldStop(); + + uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size); + + uint32_t writeMapEnd(); + + uint32_t writeListBegin(const TType elemType, + const uint32_t size); + + uint32_t writeListEnd(); + + uint32_t writeSetBegin(const TType elemType, + const uint32_t size); + + uint32_t writeSetEnd(); + + uint32_t writeBool(const bool value); + + uint32_t writeByte(const int8_t byte); + + uint32_t writeI16(const int16_t i16); + + uint32_t writeI32(const int32_t i32); + + uint32_t writeI64(const int64_t i64); + + uint32_t writeDouble(const double dub); + + uint32_t writeString(const std::string& str); + + uint32_t writeBinary(const std::string& str); + + /** + * Reading functions + */ + + + uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid); + + uint32_t readMessageEnd(); + + uint32_t readStructBegin(std::string& name); + + uint32_t readStructEnd(); + + uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId); + + uint32_t readFieldEnd(); + + uint32_t readMapBegin(TType& keyType, + TType& valType, + uint32_t& size); + + uint32_t readMapEnd(); + + uint32_t readListBegin(TType& elemType, + uint32_t& size); + + uint32_t readListEnd(); + + uint32_t readSetBegin(TType& elemType, + uint32_t& size); + + uint32_t readSetEnd(); + + uint32_t readBool(bool& value); + + uint32_t readByte(int8_t& byte); + + uint32_t readI16(int16_t& i16); + + uint32_t readI32(int32_t& i32); + + uint32_t readI64(int64_t& i64); + + uint32_t readDouble(double& dub); + + uint32_t readString(std::string& str); + + uint32_t readBinary(std::string& str); + + protected: + uint32_t readStringBody(std::string& str, int32_t sz); + + int32_t string_limit_; + int32_t container_limit_; + + // Enforce presence of version identifier + bool strict_read_; + bool strict_write_; + + // Buffer for reading strings, save for the lifetime of the protocol to + // avoid memory churn allocating memory on every string read + uint8_t* string_buf_; + int32_t string_buf_size_; + +}; + +/** + * Constructs binary protocol handlers + */ +class TBinaryProtocolFactory : public TProtocolFactory { + public: + TBinaryProtocolFactory() : + string_limit_(0), + container_limit_(0), + strict_read_(false), + strict_write_(true) {} + + TBinaryProtocolFactory(int32_t string_limit, int32_t container_limit, bool strict_read, bool strict_write) : + string_limit_(string_limit), + container_limit_(container_limit), + strict_read_(strict_read), + strict_write_(strict_write) {} + + virtual ~TBinaryProtocolFactory() {} + + void setStringSizeLimit(int32_t string_limit) { + string_limit_ = string_limit; + } + + void setContainerSizeLimit(int32_t container_limit) { + container_limit_ = container_limit; + } + + void setStrict(bool strict_read, bool strict_write) { + strict_read_ = strict_read; + strict_write_ = strict_write; + } + + boost::shared_ptr getProtocol(boost::shared_ptr trans) { + return boost::shared_ptr(new TBinaryProtocol(trans, string_limit_, container_limit_, strict_read_, strict_write_)); + } + + private: + int32_t string_limit_; + int32_t container_limit_; + bool strict_read_; + bool strict_write_; + +}; + +}}} // apache::thrift::protocol + +#endif // #ifndef _THRIFT_PROTOCOL_TBINARYPROTOCOL_H_ diff --git a/lib/cpp/src/protocol/TCompactProtocol.cpp b/lib/cpp/src/protocol/TCompactProtocol.cpp new file mode 100644 index 00000000..ce2ee54d --- /dev/null +++ b/lib/cpp/src/protocol/TCompactProtocol.cpp @@ -0,0 +1,736 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "TCompactProtocol.h" + +#include +#include + +/* + * TCompactProtocol::i*ToZigzag depend on the fact that the right shift + * operator on a signed integer is an arithmetic (sign-extending) shift. + * If this is not the case, the current implementation will not work. + * If anyone encounters this error, we can try to figure out the best + * way to implement an arithmetic right shift on their platform. + */ +#if !defined(SIGNED_RIGHT_SHIFT_IS) || !defined(ARITHMETIC_RIGHT_SHIFT) +# error "Unable to determine the behavior of a signed right shift" +#endif +#if SIGNED_RIGHT_SHIFT_IS != ARITHMETIC_RIGHT_SHIFT +# error "TCompactProtocol currenly only works if a signed right shift is arithmetic" +#endif + +#ifdef __GNUC__ +#define UNLIKELY(val) (__builtin_expect((val), 0)) +#else +#define UNLIKELY(val) (val) +#endif + +namespace apache { namespace thrift { namespace protocol { + +const int8_t TCompactProtocol::TTypeToCType[16] = { + CT_STOP, // T_STOP + 0, // unused + CT_BOOLEAN_TRUE, // T_BOOL + CT_BYTE, // T_BYTE + CT_DOUBLE, // T_DOUBLE + 0, // unused + CT_I16, // T_I16 + 0, // unused + CT_I32, // T_I32 + 0, // unused + CT_I64, // T_I64 + CT_BINARY, // T_STRING + CT_STRUCT, // T_STRUCT + CT_MAP, // T_MAP + CT_SET, // T_SET + CT_LIST, // T_LIST + }; + + +uint32_t TCompactProtocol::writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid) { + uint32_t wsize = 0; + wsize += writeByte(PROTOCOL_ID); + wsize += writeByte((VERSION_N & VERSION_MASK) | (((int32_t)messageType << TYPE_SHIFT_AMOUNT) & TYPE_MASK)); + wsize += writeVarint32(seqid); + wsize += writeString(name); + return wsize; +} + +/** + * Write a field header containing the field id and field type. If the + * difference between the current field id and the last one is small (< 15), + * then the field id will be encoded in the 4 MSB as a delta. Otherwise, the + * field id will follow the type header as a zigzag varint. + */ +uint32_t TCompactProtocol::writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId) { + if (fieldType == T_BOOL) { + booleanField_.name = name; + booleanField_.fieldType = fieldType; + booleanField_.fieldId = fieldId; + } else { + return writeFieldBeginInternal(name, fieldType, fieldId, -1); + } + return 0; +} + +/** + * Write the STOP symbol so we know there are no more fields in this struct. + */ +uint32_t TCompactProtocol::writeFieldStop() { + return writeByte(T_STOP); +} + +/** + * Write a struct begin. This doesn't actually put anything on the wire. We + * use it as an opportunity to put special placeholder markers on the field + * stack so we can get the field id deltas correct. + */ +uint32_t TCompactProtocol::writeStructBegin(const char* name) { + lastField_.push(lastFieldId_); + lastFieldId_ = 0; + return 0; +} + +/** + * Write a struct end. This doesn't actually put anything on the wire. We use + * this as an opportunity to pop the last field from the current struct off + * of the field stack. + */ +uint32_t TCompactProtocol::writeStructEnd() { + lastFieldId_ = lastField_.top(); + lastField_.pop(); + return 0; +} + +/** + * Write a List header. + */ +uint32_t TCompactProtocol::writeListBegin(const TType elemType, + const uint32_t size) { + return writeCollectionBegin(elemType, size); +} + +/** + * Write a set header. + */ +uint32_t TCompactProtocol::writeSetBegin(const TType elemType, + const uint32_t size) { + return writeCollectionBegin(elemType, size); +} + +/** + * Write a map header. If the map is empty, omit the key and value type + * headers, as we don't need any additional information to skip it. + */ +uint32_t TCompactProtocol::writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size) { + uint32_t wsize = 0; + + if (size == 0) { + wsize += writeByte(0); + } else { + wsize += writeVarint32(size); + wsize += writeByte(getCompactType(keyType) << 4 | getCompactType(valType)); + } + return wsize; +} + +/** + * Write a boolean value. Potentially, this could be a boolean field, in + * which case the field header info isn't written yet. If so, decide what the + * right type header is for the value and then write the field header. + * Otherwise, write a single byte. + */ +uint32_t TCompactProtocol::writeBool(const bool value) { + uint32_t wsize = 0; + + if (booleanField_.name != NULL) { + // we haven't written the field header yet + wsize += writeFieldBeginInternal(booleanField_.name, + booleanField_.fieldType, + booleanField_.fieldId, + value ? CT_BOOLEAN_TRUE : CT_BOOLEAN_FALSE); + booleanField_.name = NULL; + } else { + // we're not part of a field, so just write the value + wsize += writeByte(value ? CT_BOOLEAN_TRUE : CT_BOOLEAN_FALSE); + } + return wsize; +} + +uint32_t TCompactProtocol::writeByte(const int8_t byte) { + trans_->write((uint8_t*)&byte, 1); + return 1; +} + +/** + * Write an i16 as a zigzag varint. + */ +uint32_t TCompactProtocol::writeI16(const int16_t i16) { + return writeVarint32(i32ToZigzag(i16)); +} + +/** + * Write an i32 as a zigzag varint. + */ +uint32_t TCompactProtocol::writeI32(const int32_t i32) { + return writeVarint32(i32ToZigzag(i32)); +} + +/** + * Write an i64 as a zigzag varint. + */ +uint32_t TCompactProtocol::writeI64(const int64_t i64) { + return writeVarint64(i64ToZigzag(i64)); +} + +/** + * Write a double to the wire as 8 bytes. + */ +uint32_t TCompactProtocol::writeDouble(const double dub) { + BOOST_STATIC_ASSERT(sizeof(double) == sizeof(uint64_t)); + BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); + + uint64_t bits = bitwise_cast(dub); + bits = htolell(bits); + trans_->write((uint8_t*)&bits, 8); + return 8; +} + +/** + * Write a string to the wire with a varint size preceeding. + */ +uint32_t TCompactProtocol::writeString(const std::string& str) { + return writeBinary(str); +} + +uint32_t TCompactProtocol::writeBinary(const std::string& str) { + uint32_t ssize = str.size(); + uint32_t wsize = writeVarint32(ssize) + ssize; + trans_->write((uint8_t*)str.data(), ssize); + return wsize; +} + +// +// Internal Writing methods +// + +/** + * The workhorse of writeFieldBegin. It has the option of doing a + * 'type override' of the type header. This is used specifically in the + * boolean field case. + */ +int32_t TCompactProtocol::writeFieldBeginInternal(const char* name, + const TType fieldType, + const int16_t fieldId, + int8_t typeOverride) { + uint32_t wsize = 0; + + // if there's a type override, use that. + int8_t typeToWrite = (typeOverride == -1 ? getCompactType(fieldType) : typeOverride); + + // check if we can use delta encoding for the field id + if (fieldId > lastFieldId_ && fieldId - lastFieldId_ <= 15) { + // write them together + wsize += writeByte((fieldId - lastFieldId_) << 4 | typeToWrite); + } else { + // write them separate + wsize += writeByte(typeToWrite); + wsize += writeI16(fieldId); + } + + lastFieldId_ = fieldId; + return wsize; +} + +/** + * Abstract method for writing the start of lists and sets. List and sets on + * the wire differ only by the type indicator. + */ +uint32_t TCompactProtocol::writeCollectionBegin(int8_t elemType, int32_t size) { + uint32_t wsize = 0; + if (size <= 14) { + wsize += writeByte(size << 4 | getCompactType(elemType)); + } else { + wsize += writeByte(0xf0 | getCompactType(elemType)); + wsize += writeVarint32(size); + } + return wsize; +} + +/** + * Write an i32 as a varint. Results in 1-5 bytes on the wire. + */ +uint32_t TCompactProtocol::writeVarint32(uint32_t n) { + uint8_t buf[5]; + uint32_t wsize = 0; + + while (true) { + if ((n & ~0x7F) == 0) { + buf[wsize++] = (int8_t)n; + break; + } else { + buf[wsize++] = (int8_t)((n & 0x7F) | 0x80); + n >>= 7; + } + } + trans_->write(buf, wsize); + return wsize; +} + +/** + * Write an i64 as a varint. Results in 1-10 bytes on the wire. + */ +uint32_t TCompactProtocol::writeVarint64(uint64_t n) { + uint8_t buf[10]; + uint32_t wsize = 0; + + while (true) { + if ((n & ~0x7FL) == 0) { + buf[wsize++] = (int8_t)n; + break; + } else { + buf[wsize++] = (int8_t)((n & 0x7F) | 0x80); + n >>= 7; + } + } + trans_->write(buf, wsize); + return wsize; +} + +/** + * Convert l into a zigzag long. This allows negative numbers to be + * represented compactly as a varint. + */ +uint64_t TCompactProtocol::i64ToZigzag(const int64_t l) { + return (l << 1) ^ (l >> 63); +} + +/** + * Convert n into a zigzag int. This allows negative numbers to be + * represented compactly as a varint. + */ +uint32_t TCompactProtocol::i32ToZigzag(const int32_t n) { + return (n << 1) ^ (n >> 31); +} + +/** + * Given a TType value, find the appropriate TCompactProtocol.Type value + */ +int8_t TCompactProtocol::getCompactType(int8_t ttype) { + return TTypeToCType[ttype]; +} + +// +// Reading Methods +// + +/** + * Read a message header. + */ +uint32_t TCompactProtocol::readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid) { + uint32_t rsize = 0; + int8_t protocolId; + int8_t versionAndType; + int8_t version; + + rsize += readByte(protocolId); + if (protocolId != PROTOCOL_ID) { + throw TProtocolException(TProtocolException::BAD_VERSION, "Bad protocol identifier"); + } + + rsize += readByte(versionAndType); + version = (int8_t)(versionAndType & VERSION_MASK); + if (version != VERSION_N) { + throw TProtocolException(TProtocolException::BAD_VERSION, "Bad protocol version"); + } + + messageType = (TMessageType)((versionAndType >> TYPE_SHIFT_AMOUNT) & 0x03); + rsize += readVarint32(seqid); + rsize += readString(name); + + return rsize; +} + +/** + * Read a struct begin. There's nothing on the wire for this, but it is our + * opportunity to push a new struct begin marker on the field stack. + */ +uint32_t TCompactProtocol::readStructBegin(std::string& name) { + name = ""; + lastField_.push(lastFieldId_); + lastFieldId_ = 0; + return 0; +} + +/** + * Doesn't actually consume any wire data, just removes the last field for + * this struct from the field stack. + */ +uint32_t TCompactProtocol::readStructEnd() { + lastFieldId_ = lastField_.top(); + lastField_.pop(); + return 0; +} + +/** + * Read a field header off the wire. + */ +uint32_t TCompactProtocol::readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId) { + uint32_t rsize = 0; + int8_t byte; + int8_t type; + + rsize += readByte(byte); + type = (byte & 0x0f); + + // if it's a stop, then we can return immediately, as the struct is over. + if (type == T_STOP) { + fieldType = T_STOP; + fieldId = 0; + return rsize; + } + + // mask off the 4 MSB of the type header. it could contain a field id delta. + int16_t modifier = (int16_t)(((uint8_t)byte & 0xf0) >> 4); + if (modifier == 0) { + // not a delta, look ahead for the zigzag varint field id. + rsize += readI16(fieldId); + } else { + fieldId = (int16_t)(lastFieldId_ + modifier); + } + fieldType = getTType(type); + + // if this happens to be a boolean field, the value is encoded in the type + if (type == CT_BOOLEAN_TRUE || type == CT_BOOLEAN_FALSE) { + // save the boolean value in a special instance variable. + boolValue_.hasBoolValue = true; + boolValue_.boolValue = (type == CT_BOOLEAN_TRUE ? true : false); + } + + // push the new field onto the field stack so we can keep the deltas going. + lastFieldId_ = fieldId; + return rsize; +} + +/** + * Read a map header off the wire. If the size is zero, skip reading the key + * and value type. This means that 0-length maps will yield TMaps without the + * "correct" types. + */ +uint32_t TCompactProtocol::readMapBegin(TType& keyType, + TType& valType, + uint32_t& size) { + uint32_t rsize = 0; + int8_t kvType = 0; + int32_t msize = 0; + + rsize += readVarint32(msize); + if (msize != 0) + rsize += readByte(kvType); + + if (msize < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (container_limit_ && msize > container_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + + keyType = getTType((int8_t)((uint8_t)kvType >> 4)); + valType = getTType((int8_t)((uint8_t)kvType & 0xf)); + size = (uint32_t)msize; + + return rsize; +} + +/** + * Read a list header off the wire. If the list size is 0-14, the size will + * be packed into the element type header. If it's a longer list, the 4 MSB + * of the element type header will be 0xF, and a varint will follow with the + * true size. + */ +uint32_t TCompactProtocol::readListBegin(TType& elemType, + uint32_t& size) { + int8_t size_and_type; + uint32_t rsize = 0; + int32_t lsize; + + rsize += readByte(size_and_type); + + lsize = ((uint8_t)size_and_type >> 4) & 0x0f; + if (lsize == 15) { + rsize += readVarint32(lsize); + } + + if (lsize < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (container_limit_ && lsize > container_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + + elemType = getTType((int8_t)(size_and_type & 0x0f)); + size = (uint32_t)lsize; + + return rsize; +} + +/** + * Read a set header off the wire. If the set size is 0-14, the size will + * be packed into the element type header. If it's a longer set, the 4 MSB + * of the element type header will be 0xF, and a varint will follow with the + * true size. + */ +uint32_t TCompactProtocol::readSetBegin(TType& elemType, + uint32_t& size) { + return readListBegin(elemType, size); +} + +/** + * Read a boolean off the wire. If this is a boolean field, the value should + * already have been read during readFieldBegin, so we'll just consume the + * pre-stored value. Otherwise, read a byte. + */ +uint32_t TCompactProtocol::readBool(bool& value) { + if (boolValue_.hasBoolValue == true) { + value = boolValue_.boolValue; + boolValue_.hasBoolValue = false; + return 0; + } else { + int8_t val; + readByte(val); + value = (val == CT_BOOLEAN_TRUE); + return 1; + } +} + +/** + * Read a single byte off the wire. Nothing interesting here. + */ +uint32_t TCompactProtocol::readByte(int8_t& byte) { + uint8_t b[1]; + trans_->readAll(b, 1); + byte = *(int8_t*)b; + return 1; +} + +/** + * Read an i16 from the wire as a zigzag varint. + */ +uint32_t TCompactProtocol::readI16(int16_t& i16) { + int32_t value; + uint32_t rsize = readVarint32(value); + i16 = (int16_t)zigzagToI32(value); + return rsize; +} + +/** + * Read an i32 from the wire as a zigzag varint. + */ +uint32_t TCompactProtocol::readI32(int32_t& i32) { + int32_t value; + uint32_t rsize = readVarint32(value); + i32 = zigzagToI32(value); + return rsize; +} + +/** + * Read an i64 from the wire as a zigzag varint. + */ +uint32_t TCompactProtocol::readI64(int64_t& i64) { + int64_t value; + uint32_t rsize = readVarint64(value); + i64 = zigzagToI64(value); + return rsize; +} + +/** + * No magic here - just read a double off the wire. + */ +uint32_t TCompactProtocol::readDouble(double& dub) { + BOOST_STATIC_ASSERT(sizeof(double) == sizeof(uint64_t)); + BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); + + uint64_t bits; + uint8_t b[8]; + trans_->readAll(b, 8); + bits = *(uint64_t*)b; + bits = letohll(bits); + dub = bitwise_cast(bits); + return 8; +} + +uint32_t TCompactProtocol::readString(std::string& str) { + return readBinary(str); +} + +/** + * Read a byte[] from the wire. + */ +uint32_t TCompactProtocol::readBinary(std::string& str) { + int32_t rsize = 0; + int32_t size; + + rsize += readVarint32(size); + // Catch empty string case + if (size == 0) { + str = ""; + return rsize; + } + + // Catch error cases + if (size < 0) { + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } + if (string_limit_ > 0 && size > string_limit_) { + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + + // Use the heap here to prevent stack overflow for v. large strings + if (size > string_buf_size_ || string_buf_ == NULL) { + void* new_string_buf = std::realloc(string_buf_, (uint32_t)size); + if (new_string_buf == NULL) { + throw TProtocolException(TProtocolException::UNKNOWN, "Out of memory in TCompactProtocol::readString"); + } + string_buf_ = (uint8_t*)new_string_buf; + string_buf_size_ = size; + } + trans_->readAll(string_buf_, size); + str.assign((char*)string_buf_, size); + + return rsize + (uint32_t)size; +} + +/** + * Read an i32 from the wire as a varint. The MSB of each byte is set + * if there is another byte to follow. This can read up to 5 bytes. + */ +uint32_t TCompactProtocol::readVarint32(int32_t& i32) { + int64_t val; + uint32_t rsize = readVarint64(val); + i32 = (int32_t)val; + return rsize; +} + +/** + * Read an i64 from the wire as a proper varint. The MSB of each byte is set + * if there is another byte to follow. This can read up to 10 bytes. + */ +uint32_t TCompactProtocol::readVarint64(int64_t& i64) { + uint32_t rsize = 0; + uint64_t val = 0; + int shift = 0; + uint8_t buf[10]; // 64 bits / (7 bits/byte) = 10 bytes. + uint32_t buf_size = sizeof(buf); + const uint8_t* borrowed = trans_->borrow(buf, &buf_size); + + // Fast path. + if (borrowed != NULL) { + while (true) { + uint8_t byte = borrowed[rsize]; + rsize++; + val |= (uint64_t)(byte & 0x7f) << shift; + shift += 7; + if (!(byte & 0x80)) { + i64 = val; + trans_->consume(rsize); + return rsize; + } + // Have to check for invalid data so we don't crash. + if (UNLIKELY(rsize == sizeof(buf))) { + throw TProtocolException(TProtocolException::INVALID_DATA, "Variable-length int over 10 bytes."); + } + } + } + + // Slow path. + else { + while (true) { + uint8_t byte; + rsize += trans_->readAll(&byte, 1); + val |= (uint64_t)(byte & 0x7f) << shift; + shift += 7; + if (!(byte & 0x80)) { + i64 = val; + return rsize; + } + // Might as well check for invalid data on the slow path too. + if (UNLIKELY(rsize >= sizeof(buf))) { + throw TProtocolException(TProtocolException::INVALID_DATA, "Variable-length int over 10 bytes."); + } + } + } +} + +/** + * Convert from zigzag int to int. + */ +int32_t TCompactProtocol::zigzagToI32(uint32_t n) { + return (n >> 1) ^ -(n & 1); +} + +/** + * Convert from zigzag long to long. + */ +int64_t TCompactProtocol::zigzagToI64(uint64_t n) { + return (n >> 1) ^ -(n & 1); +} + +TType TCompactProtocol::getTType(int8_t type) { + switch (type) { + case T_STOP: + return T_STOP; + case CT_BOOLEAN_FALSE: + case CT_BOOLEAN_TRUE: + return T_BOOL; + case CT_BYTE: + return T_BYTE; + case CT_I16: + return T_I16; + case CT_I32: + return T_I32; + case CT_I64: + return T_I64; + case CT_DOUBLE: + return T_DOUBLE; + case CT_BINARY: + return T_STRING; + case CT_LIST: + return T_LIST; + case CT_SET: + return T_SET; + case CT_MAP: + return T_MAP; + case CT_STRUCT: + return T_STRUCT; + default: + throw TException("don't know what type: " + type); + } + return T_STOP; +} + +}}} // apache::thrift::protocol diff --git a/lib/cpp/src/protocol/TCompactProtocol.h b/lib/cpp/src/protocol/TCompactProtocol.h new file mode 100644 index 00000000..b4e06f0a --- /dev/null +++ b/lib/cpp/src/protocol/TCompactProtocol.h @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TCOMPACTPROTOCOL_H_ +#define _THRIFT_PROTOCOL_TCOMPACTPROTOCOL_H_ 1 + +#include "TProtocol.h" + +#include +#include + +namespace apache { namespace thrift { namespace protocol { + +/** + * C++ Implementation of the Compact Protocol as described in THRIFT-110 + */ +class TCompactProtocol : public TProtocol { + + protected: + static const int8_t PROTOCOL_ID = 0x82; + static const int8_t VERSION_N = 1; + static const int8_t VERSION_MASK = 0x1f; // 0001 1111 + static const int8_t TYPE_MASK = 0xE0; // 1110 0000 + static const int32_t TYPE_SHIFT_AMOUNT = 5; + + /** + * (Writing) If we encounter a boolean field begin, save the TField here + * so it can have the value incorporated. + */ + struct { + const char* name; + TType fieldType; + int16_t fieldId; + } booleanField_; + + /** + * (Reading) If we read a field header, and it's a boolean field, save + * the boolean value here so that readBool can use it. + */ + struct { + bool hasBoolValue; + bool boolValue; + } boolValue_; + + /** + * Used to keep track of the last field for the current and previous structs, + * so we can do the delta stuff. + */ + + std::stack lastField_; + int16_t lastFieldId_; + + enum Types { + CT_STOP = 0x00, + CT_BOOLEAN_TRUE = 0x01, + CT_BOOLEAN_FALSE = 0x02, + CT_BYTE = 0x03, + CT_I16 = 0x04, + CT_I32 = 0x05, + CT_I64 = 0x06, + CT_DOUBLE = 0x07, + CT_BINARY = 0x08, + CT_LIST = 0x09, + CT_SET = 0x0A, + CT_MAP = 0x0B, + CT_STRUCT = 0x0C, + }; + + static const int8_t TTypeToCType[16]; + + public: + TCompactProtocol(boost::shared_ptr trans) : + TProtocol(trans), + lastFieldId_(0), + string_limit_(0), + string_buf_(NULL), + string_buf_size_(0), + container_limit_(0) { + booleanField_.name = NULL; + boolValue_.hasBoolValue = false; + } + + TCompactProtocol(boost::shared_ptr trans, + int32_t string_limit, + int32_t container_limit) : + TProtocol(trans), + lastFieldId_(0), + string_limit_(string_limit), + string_buf_(NULL), + string_buf_size_(0), + container_limit_(container_limit) { + booleanField_.name = NULL; + boolValue_.hasBoolValue = false; + } + + + + /** + * Writing functions + */ + + virtual uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid); + + uint32_t writeStructBegin(const char* name); + + uint32_t writeStructEnd(); + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId); + + uint32_t writeFieldStop(); + + uint32_t writeListBegin(const TType elemType, + const uint32_t size); + + uint32_t writeSetBegin(const TType elemType, + const uint32_t size); + + virtual uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size); + + uint32_t writeBool(const bool value); + + uint32_t writeByte(const int8_t byte); + + uint32_t writeI16(const int16_t i16); + + uint32_t writeI32(const int32_t i32); + + uint32_t writeI64(const int64_t i64); + + uint32_t writeDouble(const double dub); + + uint32_t writeString(const std::string& str); + + uint32_t writeBinary(const std::string& str); + + /** + * These methods are called by structs, but don't actually have any wired + * output or purpose + */ + virtual uint32_t writeMessageEnd() { return 0; } + uint32_t writeMapEnd() { return 0; } + uint32_t writeListEnd() { return 0; } + uint32_t writeSetEnd() { return 0; } + uint32_t writeFieldEnd() { return 0; } + + protected: + int32_t writeFieldBeginInternal(const char* name, + const TType fieldType, + const int16_t fieldId, + int8_t typeOverride); + uint32_t writeCollectionBegin(int8_t elemType, int32_t size); + uint32_t writeVarint32(uint32_t n); + uint32_t writeVarint64(uint64_t n); + uint64_t i64ToZigzag(const int64_t l); + uint32_t i32ToZigzag(const int32_t n); + inline int8_t getCompactType(int8_t ttype); + + public: + uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid); + + uint32_t readStructBegin(std::string& name); + + uint32_t readStructEnd(); + + uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId); + + uint32_t readMapBegin(TType& keyType, + TType& valType, + uint32_t& size); + + uint32_t readListBegin(TType& elemType, + uint32_t& size); + + uint32_t readSetBegin(TType& elemType, + uint32_t& size); + + uint32_t readBool(bool& value); + + uint32_t readByte(int8_t& byte); + + uint32_t readI16(int16_t& i16); + + uint32_t readI32(int32_t& i32); + + uint32_t readI64(int64_t& i64); + + uint32_t readDouble(double& dub); + + uint32_t readString(std::string& str); + + uint32_t readBinary(std::string& str); + + /* + *These methods are here for the struct to call, but don't have any wire + * encoding. + */ + uint32_t readMessageEnd() { return 0; } + uint32_t readFieldEnd() { return 0; } + uint32_t readMapEnd() { return 0; } + uint32_t readListEnd() { return 0; } + uint32_t readSetEnd() { return 0; } + + protected: + uint32_t readVarint32(int32_t& i32); + uint32_t readVarint64(int64_t& i64); + int32_t zigzagToI32(uint32_t n); + int64_t zigzagToI64(uint64_t n); + TType getTType(int8_t type); + + // Buffer for reading strings, save for the lifetime of the protocol to + // avoid memory churn allocating memory on every string read + int32_t string_limit_; + uint8_t* string_buf_; + int32_t string_buf_size_; + int32_t container_limit_; +}; + +/** + * Constructs compact protocol handlers + */ +class TCompactProtocolFactory : public TProtocolFactory { + public: + TCompactProtocolFactory() : + string_limit_(0), + container_limit_(0) {} + + TCompactProtocolFactory(int32_t string_limit, int32_t container_limit) : + string_limit_(string_limit), + container_limit_(container_limit) {} + + virtual ~TCompactProtocolFactory() {} + + void setStringSizeLimit(int32_t string_limit) { + string_limit_ = string_limit; + } + + void setContainerSizeLimit(int32_t container_limit) { + container_limit_ = container_limit; + } + + boost::shared_ptr getProtocol(boost::shared_ptr trans) { + return boost::shared_ptr(new TCompactProtocol(trans, string_limit_, container_limit_)); + } + + private: + int32_t string_limit_; + int32_t container_limit_; + +}; + +}}} // apache::thrift::protocol + +#endif diff --git a/lib/cpp/src/protocol/TDebugProtocol.cpp b/lib/cpp/src/protocol/TDebugProtocol.cpp new file mode 100644 index 00000000..40aa36ba --- /dev/null +++ b/lib/cpp/src/protocol/TDebugProtocol.cpp @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "TDebugProtocol.h" + +#include +#include +#include +#include +#include +#include + +using std::string; + + +static string byte_to_hex(const uint8_t byte) { + char buf[3]; + int ret = std::sprintf(buf, "%02x", (int)byte); + assert(ret == 2); + assert(buf[2] == '\0'); + return buf; +} + + +namespace apache { namespace thrift { namespace protocol { + +string TDebugProtocol::fieldTypeName(TType type) { + switch (type) { + case T_STOP : return "stop" ; + case T_VOID : return "void" ; + case T_BOOL : return "bool" ; + case T_BYTE : return "byte" ; + case T_I16 : return "i16" ; + case T_I32 : return "i32" ; + case T_U64 : return "u64" ; + case T_I64 : return "i64" ; + case T_DOUBLE : return "double" ; + case T_STRING : return "string" ; + case T_STRUCT : return "struct" ; + case T_MAP : return "map" ; + case T_SET : return "set" ; + case T_LIST : return "list" ; + case T_UTF8 : return "utf8" ; + case T_UTF16 : return "utf16" ; + default: return "unknown"; + } +} + +void TDebugProtocol::indentUp() { + indent_str_ += string(indent_inc, ' '); +} + +void TDebugProtocol::indentDown() { + if (indent_str_.length() < (string::size_type)indent_inc) { + throw TProtocolException(TProtocolException::INVALID_DATA); + } + indent_str_.erase(indent_str_.length() - indent_inc); +} + +uint32_t TDebugProtocol::writePlain(const string& str) { + trans_->write((uint8_t*)str.data(), str.length()); + return str.length(); +} + +uint32_t TDebugProtocol::writeIndented(const string& str) { + trans_->write((uint8_t*)indent_str_.data(), indent_str_.length()); + trans_->write((uint8_t*)str.data(), str.length()); + return indent_str_.length() + str.length(); +} + +uint32_t TDebugProtocol::startItem() { + uint32_t size; + + switch (write_state_.back()) { + case UNINIT: + // XXX figure out what to do here. + //throw TProtocolException(TProtocolException::INVALID_DATA); + //return writeIndented(str); + return 0; + case STRUCT: + return 0; + case SET: + return writeIndented(""); + case MAP_KEY: + return writeIndented(""); + case MAP_VALUE: + return writePlain(" -> "); + case LIST: + size = writeIndented( + "[" + boost::lexical_cast(list_idx_.back()) + "] = "); + list_idx_.back()++; + return size; + default: + throw std::logic_error("Invalid enum value."); + } +} + +uint32_t TDebugProtocol::endItem() { + //uint32_t size; + + switch (write_state_.back()) { + case UNINIT: + // XXX figure out what to do here. + //throw TProtocolException(TProtocolException::INVALID_DATA); + //return writeIndented(str); + return 0; + case STRUCT: + return writePlain(",\n"); + case SET: + return writePlain(",\n"); + case MAP_KEY: + write_state_.back() = MAP_VALUE; + return 0; + case MAP_VALUE: + write_state_.back() = MAP_KEY; + return writePlain(",\n"); + case LIST: + return writePlain(",\n"); + default: + throw std::logic_error("Invalid enum value."); + } +} + +uint32_t TDebugProtocol::writeItem(const std::string& str) { + uint32_t size = 0; + size += startItem(); + size += writePlain(str); + size += endItem(); + return size; +} + +uint32_t TDebugProtocol::writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid) { + string mtype; + switch (messageType) { + case T_CALL : mtype = "call" ; break; + case T_REPLY : mtype = "reply" ; break; + case T_EXCEPTION : mtype = "exn" ; break; + } + + uint32_t size = writeIndented("(" + mtype + ") " + name + "("); + indentUp(); + return size; +} + +uint32_t TDebugProtocol::writeMessageEnd() { + indentDown(); + return writeIndented(")\n"); +} + +uint32_t TDebugProtocol::writeStructBegin(const char* name) { + uint32_t size = 0; + size += startItem(); + size += writePlain(string(name) + " {\n"); + indentUp(); + write_state_.push_back(STRUCT); + return size; +} + +uint32_t TDebugProtocol::writeStructEnd() { + indentDown(); + write_state_.pop_back(); + uint32_t size = 0; + size += writeIndented("}"); + size += endItem(); + return size; +} + +uint32_t TDebugProtocol::writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId) { + // sprintf(id_str, "%02d", fieldId); + string id_str = boost::lexical_cast(fieldId); + if (id_str.length() == 1) id_str = '0' + id_str; + + return writeIndented( + id_str + ": " + + name + " (" + + fieldTypeName(fieldType) + ") = "); +} + +uint32_t TDebugProtocol::writeFieldEnd() { + assert(write_state_.back() == STRUCT); + return 0; +} + +uint32_t TDebugProtocol::writeFieldStop() { + return 0; + //writeIndented("***STOP***\n"); +} + +uint32_t TDebugProtocol::writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size) { + // TODO(dreiss): Optimize short maps? + uint32_t bsize = 0; + bsize += startItem(); + bsize += writePlain( + "map<" + fieldTypeName(keyType) + "," + fieldTypeName(valType) + ">" + "[" + boost::lexical_cast(size) + "] {\n"); + indentUp(); + write_state_.push_back(MAP_KEY); + return bsize; +} + +uint32_t TDebugProtocol::writeMapEnd() { + indentDown(); + write_state_.pop_back(); + uint32_t size = 0; + size += writeIndented("}"); + size += endItem(); + return size; +} + +uint32_t TDebugProtocol::writeListBegin(const TType elemType, + const uint32_t size) { + // TODO(dreiss): Optimize short arrays. + uint32_t bsize = 0; + bsize += startItem(); + bsize += writePlain( + "list<" + fieldTypeName(elemType) + ">" + "[" + boost::lexical_cast(size) + "] {\n"); + indentUp(); + write_state_.push_back(LIST); + list_idx_.push_back(0); + return bsize; +} + +uint32_t TDebugProtocol::writeListEnd() { + indentDown(); + write_state_.pop_back(); + list_idx_.pop_back(); + uint32_t size = 0; + size += writeIndented("}"); + size += endItem(); + return size; +} + +uint32_t TDebugProtocol::writeSetBegin(const TType elemType, + const uint32_t size) { + // TODO(dreiss): Optimize short sets. + uint32_t bsize = 0; + bsize += startItem(); + bsize += writePlain( + "set<" + fieldTypeName(elemType) + ">" + "[" + boost::lexical_cast(size) + "] {\n"); + indentUp(); + write_state_.push_back(SET); + return bsize; +} + +uint32_t TDebugProtocol::writeSetEnd() { + indentDown(); + write_state_.pop_back(); + uint32_t size = 0; + size += writeIndented("}"); + size += endItem(); + return size; +} + +uint32_t TDebugProtocol::writeBool(const bool value) { + return writeItem(value ? "true" : "false"); +} + +uint32_t TDebugProtocol::writeByte(const int8_t byte) { + return writeItem("0x" + byte_to_hex(byte)); +} + +uint32_t TDebugProtocol::writeI16(const int16_t i16) { + return writeItem(boost::lexical_cast(i16)); +} + +uint32_t TDebugProtocol::writeI32(const int32_t i32) { + return writeItem(boost::lexical_cast(i32)); +} + +uint32_t TDebugProtocol::writeI64(const int64_t i64) { + return writeItem(boost::lexical_cast(i64)); +} + +uint32_t TDebugProtocol::writeDouble(const double dub) { + return writeItem(boost::lexical_cast(dub)); +} + + +uint32_t TDebugProtocol::writeString(const string& str) { + // XXX Raw/UTF-8? + + string to_show = str; + if (to_show.length() > (string::size_type)string_limit_) { + to_show = str.substr(0, string_prefix_size_); + to_show += "[...](" + boost::lexical_cast(str.length()) + ")"; + } + + string output = "\""; + + for (string::const_iterator it = to_show.begin(); it != to_show.end(); ++it) { + if (*it == '\\') { + output += "\\\\"; + } else if (*it == '"') { + output += "\\\""; + } else if (std::isprint(*it)) { + output += *it; + } else { + switch (*it) { + case '\a': output += "\\a"; break; + case '\b': output += "\\b"; break; + case '\f': output += "\\f"; break; + case '\n': output += "\\n"; break; + case '\r': output += "\\r"; break; + case '\t': output += "\\t"; break; + case '\v': output += "\\v"; break; + default: + output += "\\x"; + output += byte_to_hex(*it); + } + } + } + + output += '\"'; + return writeItem(output); +} + +uint32_t TDebugProtocol::writeBinary(const string& str) { + // XXX Hex? + return TDebugProtocol::writeString(str); +} + +}}} // apache::thrift::protocol diff --git a/lib/cpp/src/protocol/TDebugProtocol.h b/lib/cpp/src/protocol/TDebugProtocol.h new file mode 100644 index 00000000..ab69e0ca --- /dev/null +++ b/lib/cpp/src/protocol/TDebugProtocol.h @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TDEBUGPROTOCOL_H_ +#define _THRIFT_PROTOCOL_TDEBUGPROTOCOL_H_ 1 + +#include "TProtocol.h" +#include "TOneWayProtocol.h" + +#include + +namespace apache { namespace thrift { namespace protocol { + +/* + +!!! EXPERIMENTAL CODE !!! + +This protocol is very much a work in progress. +It doesn't handle many cases properly. +It throws exceptions in many cases. +It probably segfaults in many cases. +Bug reports and feature requests are welcome. +Complaints are not. :R + +*/ + + +/** + * Protocol that prints the payload in a nice human-readable format. + * Reading from this protocol is not supported. + * + */ +class TDebugProtocol : public TWriteOnlyProtocol { + private: + enum write_state_t + { UNINIT + , STRUCT + , LIST + , SET + , MAP_KEY + , MAP_VALUE + }; + + public: + TDebugProtocol(boost::shared_ptr trans) + : TWriteOnlyProtocol(trans, "TDebugProtocol") + , string_limit_(DEFAULT_STRING_LIMIT) + , string_prefix_size_(DEFAULT_STRING_PREFIX_SIZE) + { + write_state_.push_back(UNINIT); + } + + static const int32_t DEFAULT_STRING_LIMIT = 256; + static const int32_t DEFAULT_STRING_PREFIX_SIZE = 16; + + void setStringSizeLimit(int32_t string_limit) { + string_limit_ = string_limit; + } + + void setStringPrefixSize(int32_t string_prefix_size) { + string_prefix_size_ = string_prefix_size; + } + + + virtual uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid); + + virtual uint32_t writeMessageEnd(); + + + uint32_t writeStructBegin(const char* name); + + uint32_t writeStructEnd(); + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId); + + uint32_t writeFieldEnd(); + + uint32_t writeFieldStop(); + + uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size); + + uint32_t writeMapEnd(); + + uint32_t writeListBegin(const TType elemType, + const uint32_t size); + + uint32_t writeListEnd(); + + uint32_t writeSetBegin(const TType elemType, + const uint32_t size); + + uint32_t writeSetEnd(); + + uint32_t writeBool(const bool value); + + uint32_t writeByte(const int8_t byte); + + uint32_t writeI16(const int16_t i16); + + uint32_t writeI32(const int32_t i32); + + uint32_t writeI64(const int64_t i64); + + uint32_t writeDouble(const double dub); + + uint32_t writeString(const std::string& str); + + uint32_t writeBinary(const std::string& str); + + + private: + void indentUp(); + void indentDown(); + uint32_t writePlain(const std::string& str); + uint32_t writeIndented(const std::string& str); + uint32_t startItem(); + uint32_t endItem(); + uint32_t writeItem(const std::string& str); + + static std::string fieldTypeName(TType type); + + int32_t string_limit_; + int32_t string_prefix_size_; + + std::string indent_str_; + static const int indent_inc = 2; + + std::vector write_state_; + std::vector list_idx_; +}; + +/** + * Constructs debug protocol handlers + */ +class TDebugProtocolFactory : public TProtocolFactory { + public: + TDebugProtocolFactory() {} + virtual ~TDebugProtocolFactory() {} + + boost::shared_ptr getProtocol(boost::shared_ptr trans) { + return boost::shared_ptr(new TDebugProtocol(trans)); + } + +}; + +}}} // apache::thrift::protocol + + +// TODO(dreiss): Move (part of) ThriftDebugString into a .cpp file and remove this. +#include + +namespace apache { namespace thrift { + +template +std::string ThriftDebugString(const ThriftStruct& ts) { + using namespace apache::thrift::transport; + using namespace apache::thrift::protocol; + TMemoryBuffer* buffer = new TMemoryBuffer; + boost::shared_ptr trans(buffer); + TDebugProtocol protocol(trans); + + ts.write(&protocol); + + uint8_t* buf; + uint32_t size; + buffer->getBuffer(&buf, &size); + return std::string((char*)buf, (unsigned int)size); +} + +// TODO(dreiss): This is badly broken. Don't use it unless you are me. +#if 0 +template +std::string DebugString(const std::vector& vec) { + using namespace apache::thrift::transport; + using namespace apache::thrift::protocol; + TMemoryBuffer* buffer = new TMemoryBuffer; + boost::shared_ptr trans(buffer); + TDebugProtocol protocol(trans); + + // I am gross! + protocol.writeStructBegin("SomeRandomVector"); + + // TODO: Fix this with a trait. + protocol.writeListBegin((TType)99, vec.size()); + typename std::vector::const_iterator it; + for (it = vec.begin(); it != vec.end(); ++it) { + it->write(&protocol); + } + protocol.writeListEnd(); + + uint8_t* buf; + uint32_t size; + buffer->getBuffer(&buf, &size); + return std::string((char*)buf, (unsigned int)size); +} +#endif // 0 + +}} // apache::thrift + + +#endif // #ifndef _THRIFT_PROTOCOL_TDEBUGPROTOCOL_H_ + + diff --git a/lib/cpp/src/protocol/TDenseProtocol.cpp b/lib/cpp/src/protocol/TDenseProtocol.cpp new file mode 100644 index 00000000..8e76dc47 --- /dev/null +++ b/lib/cpp/src/protocol/TDenseProtocol.cpp @@ -0,0 +1,762 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + +IMPLEMENTATION DETAILS + +TDenseProtocol was designed to have a smaller serialized form than +TBinaryProtocol. This is accomplished using two techniques. The first is +variable-length integer encoding. We use the same technique that the Standard +MIDI File format uses for "variable-length quantities" +(http://en.wikipedia.org/wiki/Variable-length_quantity). +All integers (including i16, but not byte) are first cast to uint64_t, +then written out as variable-length quantities. This has the unfortunate side +effect that all negative numbers require 10 bytes, but negative numbers tend +to be far less common than positive ones. + +The second technique eliminating the field ids used by TBinaryProtocol. This +decision required support from the Thrift compiler and also sacrifices some of +the backward and forward compatibility of TBinaryProtocol. + +We considered implementing this technique by generating separate readers and +writers for the dense protocol (this is how Pillar, Thrift's predecessor, +worked), but this idea had a few problems: +- Our abstractions go out the window. +- We would have to maintain a second code generator. +- Preserving compatibility with old versions of the structures would be a + nightmare. + +Therefore, we chose an alternate implementation that stored the description of +the data neither in the data itself (like TBinaryProtocol) nor in the +serialization code (like Pillar), but instead in a separate data structure, +called a TypeSpec. TypeSpecs are generated by the Thrift compiler +(specifically in the t_cpp_generator), and their structure should be +documented there (TODO(dreiss): s/should be/is/). + +We maintain a stack of TypeSpecs within the protocol so it knows where the +generated code is in the reading/writing process. For example, if we are +writing an i32 contained in a struct bar, contained in a struct foo, then the +stack would look like: TOP , i32 , struct bar , struct foo , BOTTOM. +The following invariant: whenever we are about to read/write an object +(structBegin, containerBegin, or a scalar), the TypeSpec on the top of the +stack must match the type being read/written. The main reasons that this +invariant must be maintained is that if we ever start reading a structure, we +must have its exact TypeSpec in order to pass the right tags to the +deserializer. + +We use the following strategies for maintaining this invariant: + +- For structures, we have a separate stack of indexes, one for each structure + on the TypeSpec stack. These are indexes into the list of fields in the + structure's TypeSpec. When we {read,write}FieldBegin, we push on the + TypeSpec for the field. +- When we begin writing a list or set, we push on the TypeSpec for the + element type. +- For maps, we have a separate stack of booleans, one for each map on the + TypeSpec stack. The boolean is true if we are writing the key for that + map, and false if we are writing the value. Maps are the trickiest case + because the generated code does not call any protocol method between + the key and the value. As a result, we potentially have to switch + between map key state and map value state after reading/writing any object. +- This job is handled by the stateTransition method. It is called after + reading/writing every object. It pops the current TypeSpec off the stack, + then optionally pushes a new one on, depending on what the next TypeSpec is. + If it is a struct, the job is left to the next writeFieldBegin. If it is a + set or list, the just-popped typespec is pushed back on. If it is a map, + the top of the key/value stack is toggled, and the appropriate TypeSpec + is pushed. + +Optional fields are a little tricky also. We write a zero byte if they are +absent and prefix them with an 0x01 byte if they are present +*/ + +#define __STDC_LIMIT_MACROS +#include +#include "TDenseProtocol.h" +#include "TReflectionLocal.h" + +// Leaving this on for now. Disabling it will turn off asserts, which should +// give a performance boost. When we have *really* thorough test cases, +// we should drop this. +#define DEBUG_TDENSEPROTOCOL + +// NOTE: Assertions should *only* be used to detect bugs in code, +// either in TDenseProtocol itself, or in code using it. +// (For example, using the wrong TypeSpec.) +// Invalid data should NEVER cause an assertion failure, +// no matter how grossly corrupted, nor how ingeniously crafted. +#ifdef DEBUG_TDENSEPROTOCOL +#undef NDEBUG +#else +#define NDEBUG +#endif +#include + +using std::string; + +#ifdef __GNUC__ +#define UNLIKELY(val) (__builtin_expect((val), 0)) +#else +#define UNLIKELY(val) (val) +#endif + +namespace apache { namespace thrift { namespace protocol { + +const int TDenseProtocol::FP_PREFIX_LEN = + apache::thrift::reflection::local::FP_PREFIX_LEN; + +// Top TypeSpec. TypeSpec of the structure being encoded. +#define TTS (ts_stack_.back()) // type = TypeSpec* +// InDeX. Index into TTS of the current/next field to encode. +#define IDX (idx_stack_.back()) // type = int +// Field TypeSpec. TypeSpec of the current/next field to encode. +#define FTS (TTS->tstruct.specs[IDX]) // type = TypeSpec* +// Field MeTa. Metadata of the current/next field to encode. +#define FMT (TTS->tstruct.metas[IDX]) // type = FieldMeta +// SubType 1/2. TypeSpec of the first/second subtype of this container. +#define ST1 (TTS->tcontainer.subtype1) +#define ST2 (TTS->tcontainer.subtype2) + + +/** + * Checks that @c ttype is indeed the ttype that we should be writing, + * according to our typespec. Aborts if the test fails and debugging in on. + */ +inline void TDenseProtocol::checkTType(const TType ttype) { + assert(!ts_stack_.empty()); + assert(TTS->ttype == ttype); +} + +/** + * Makes sure that the TypeSpec stack is correct for the next object. + * See top-of-file comments. + */ +inline void TDenseProtocol::stateTransition() { + TypeSpec* old_tts = ts_stack_.back(); + ts_stack_.pop_back(); + + // If this is the end of the top-level write, we should have just popped + // the TypeSpec passed to the constructor. + if (ts_stack_.empty()) { + assert(old_tts = type_spec_); + return; + } + + switch (TTS->ttype) { + + case T_STRUCT: + assert(old_tts == FTS); + break; + + case T_LIST: + case T_SET: + assert(old_tts == ST1); + ts_stack_.push_back(old_tts); + break; + + case T_MAP: + assert(old_tts == (mkv_stack_.back() ? ST1 : ST2)); + mkv_stack_.back() = !mkv_stack_.back(); + ts_stack_.push_back(mkv_stack_.back() ? ST1 : ST2); + break; + + default: + assert(!"Invalid TType in stateTransition."); + break; + + } +} + + +/* + * Variable-length quantity functions. + */ + +inline uint32_t TDenseProtocol::vlqRead(uint64_t& vlq) { + uint32_t used = 0; + uint64_t val = 0; + uint8_t buf[10]; // 64 bits / (7 bits/byte) = 10 bytes. + uint32_t buf_size = sizeof(buf); + const uint8_t* borrowed = trans_->borrow(buf, &buf_size); + + // Fast path. TODO(dreiss): Make it faster. + if (borrowed != NULL) { + while (true) { + uint8_t byte = borrowed[used]; + used++; + val = (val << 7) | (byte & 0x7f); + if (!(byte & 0x80)) { + vlq = val; + trans_->consume(used); + return used; + } + // Have to check for invalid data so we don't crash. + if (UNLIKELY(used == sizeof(buf))) { + resetState(); + throw TProtocolException(TProtocolException::INVALID_DATA, "Variable-length int over 10 bytes."); + } + } + } + + // Slow path. + else { + while (true) { + uint8_t byte; + used += trans_->readAll(&byte, 1); + val = (val << 7) | (byte & 0x7f); + if (!(byte & 0x80)) { + vlq = val; + return used; + } + // Might as well check for invalid data on the slow path too. + if (UNLIKELY(used >= sizeof(buf))) { + resetState(); + throw TProtocolException(TProtocolException::INVALID_DATA, "Variable-length int over 10 bytes."); + } + } + } +} + +inline uint32_t TDenseProtocol::vlqWrite(uint64_t vlq) { + uint8_t buf[10]; // 64 bits / (7 bits/byte) = 10 bytes. + int32_t pos = sizeof(buf) - 1; + + // Write the thing from back to front. + buf[pos] = vlq & 0x7f; + vlq >>= 7; + pos--; + + while (vlq > 0) { + assert(pos >= 0); + buf[pos] = (vlq | 0x80); + vlq >>= 7; + pos--; + } + + // Back up one step before writing. + pos++; + + trans_->write(buf+pos, sizeof(buf) - pos); + return sizeof(buf) - pos; +} + + + +/* + * Writing functions. + */ + +uint32_t TDenseProtocol::writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid) { + throw TApplicationException("TDenseProtocol doesn't work with messages (yet)."); + + int32_t version = (VERSION_2) | ((int32_t)messageType); + uint32_t wsize = 0; + wsize += subWriteI32(version); + wsize += subWriteString(name); + wsize += subWriteI32(seqid); + return wsize; +} + +uint32_t TDenseProtocol::writeMessageEnd() { + return 0; +} + +uint32_t TDenseProtocol::writeStructBegin(const char* name) { + uint32_t xfer = 0; + + // The TypeSpec stack should be empty if this is the top-level read/write. + // If it is, we push the TypeSpec passed to the constructor. + if (ts_stack_.empty()) { + assert(standalone_); + + if (type_spec_ == NULL) { + resetState(); + throw TApplicationException("TDenseProtocol: No type specified."); + } else { + assert(type_spec_->ttype == T_STRUCT); + ts_stack_.push_back(type_spec_); + // Write out a prefix of the structure fingerprint. + trans_->write(type_spec_->fp_prefix, FP_PREFIX_LEN); + xfer += FP_PREFIX_LEN; + } + } + + // We need a new field index for this structure. + idx_stack_.push_back(0); + return 0; +} + +uint32_t TDenseProtocol::writeStructEnd() { + idx_stack_.pop_back(); + stateTransition(); + return 0; +} + +uint32_t TDenseProtocol::writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId) { + uint32_t xfer = 0; + + // Skip over optional fields. + while (FMT.tag != fieldId) { + // TODO(dreiss): Old meta here. + assert(FTS->ttype != T_STOP); + assert(FMT.is_optional); + // Write a zero byte so the reader can skip it. + xfer += subWriteBool(false); + // And advance to the next field. + IDX++; + } + + // TODO(dreiss): give a better exception. + assert(FTS->ttype == fieldType); + + if (FMT.is_optional) { + subWriteBool(true); + xfer += 1; + } + + // writeFieldStop shares all lot of logic up to this point. + // Instead of replicating it all, we just call this method from that one + // and use a gross special case here. + if (UNLIKELY(FTS->ttype != T_STOP)) { + // For normal fields, push the TypeSpec that we're about to use. + ts_stack_.push_back(FTS); + } + return xfer; +} + +uint32_t TDenseProtocol::writeFieldEnd() { + // Just move on to the next field. + IDX++; + return 0; +} + +uint32_t TDenseProtocol::writeFieldStop() { + return TDenseProtocol::writeFieldBegin("", T_STOP, 0); +} + +uint32_t TDenseProtocol::writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size) { + checkTType(T_MAP); + + assert(keyType == ST1->ttype); + assert(valType == ST2->ttype); + + ts_stack_.push_back(ST1); + mkv_stack_.push_back(true); + + return subWriteI32((int32_t)size); +} + +uint32_t TDenseProtocol::writeMapEnd() { + // Pop off the value type, as well as our entry in the map key/value stack. + // stateTransition takes care of popping off our TypeSpec. + ts_stack_.pop_back(); + mkv_stack_.pop_back(); + stateTransition(); + return 0; +} + +uint32_t TDenseProtocol::writeListBegin(const TType elemType, + const uint32_t size) { + checkTType(T_LIST); + + assert(elemType == ST1->ttype); + ts_stack_.push_back(ST1); + return subWriteI32((int32_t)size); +} + +uint32_t TDenseProtocol::writeListEnd() { + // Pop off the element type. stateTransition takes care of popping off ours. + ts_stack_.pop_back(); + stateTransition(); + return 0; +} + +uint32_t TDenseProtocol::writeSetBegin(const TType elemType, + const uint32_t size) { + checkTType(T_SET); + + assert(elemType == ST1->ttype); + ts_stack_.push_back(ST1); + return subWriteI32((int32_t)size); +} + +uint32_t TDenseProtocol::writeSetEnd() { + // Pop off the element type. stateTransition takes care of popping off ours. + ts_stack_.pop_back(); + stateTransition(); + return 0; +} + +uint32_t TDenseProtocol::writeBool(const bool value) { + checkTType(T_BOOL); + stateTransition(); + return TBinaryProtocol::writeBool(value); +} + +uint32_t TDenseProtocol::writeByte(const int8_t byte) { + checkTType(T_BYTE); + stateTransition(); + return TBinaryProtocol::writeByte(byte); +} + +uint32_t TDenseProtocol::writeI16(const int16_t i16) { + checkTType(T_I16); + stateTransition(); + return vlqWrite(i16); +} + +uint32_t TDenseProtocol::writeI32(const int32_t i32) { + checkTType(T_I32); + stateTransition(); + return vlqWrite(i32); +} + +uint32_t TDenseProtocol::writeI64(const int64_t i64) { + checkTType(T_I64); + stateTransition(); + return vlqWrite(i64); +} + +uint32_t TDenseProtocol::writeDouble(const double dub) { + checkTType(T_DOUBLE); + stateTransition(); + return TBinaryProtocol::writeDouble(dub); +} + +uint32_t TDenseProtocol::writeString(const std::string& str) { + checkTType(T_STRING); + stateTransition(); + return subWriteString(str); +} + +uint32_t TDenseProtocol::writeBinary(const std::string& str) { + return TDenseProtocol::writeString(str); +} + +inline uint32_t TDenseProtocol::subWriteI32(const int32_t i32) { + return vlqWrite(i32); +} + +uint32_t TDenseProtocol::subWriteString(const std::string& str) { + uint32_t size = str.size(); + uint32_t xfer = subWriteI32((int32_t)size); + if (size > 0) { + trans_->write((uint8_t*)str.data(), size); + } + return xfer + size; +} + + + +/* + * Reading functions + * + * These have a lot of the same logic as the writing functions, so if + * something is confusing, look for comments in the corresponding writer. + */ + +uint32_t TDenseProtocol::readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid) { + throw TApplicationException("TDenseProtocol doesn't work with messages (yet)."); + + uint32_t xfer = 0; + int32_t sz; + xfer += subReadI32(sz); + + if (sz < 0) { + // Check for correct version number + int32_t version = sz & VERSION_MASK; + if (version != VERSION_2) { + throw TProtocolException(TProtocolException::BAD_VERSION, "Bad version identifier"); + } + messageType = (TMessageType)(sz & 0x000000ff); + xfer += subReadString(name); + xfer += subReadI32(seqid); + } else { + throw TProtocolException(TProtocolException::BAD_VERSION, "No version identifier... old protocol client in strict mode?"); + } + return xfer; +} + +uint32_t TDenseProtocol::readMessageEnd() { + return 0; +} + +uint32_t TDenseProtocol::readStructBegin(string& name) { + uint32_t xfer = 0; + + if (ts_stack_.empty()) { + assert(standalone_); + + if (type_spec_ == NULL) { + resetState(); + throw TApplicationException("TDenseProtocol: No type specified."); + } else { + assert(type_spec_->ttype == T_STRUCT); + ts_stack_.push_back(type_spec_); + + // Check the fingerprint prefix. + uint8_t buf[FP_PREFIX_LEN]; + xfer += trans_->read(buf, FP_PREFIX_LEN); + if (std::memcmp(buf, type_spec_->fp_prefix, FP_PREFIX_LEN) != 0) { + resetState(); + throw TProtocolException(TProtocolException::INVALID_DATA, + "Fingerprint in data does not match type_spec."); + } + } + } + + // We need a new field index for this structure. + idx_stack_.push_back(0); + return 0; +} + +uint32_t TDenseProtocol::readStructEnd() { + idx_stack_.pop_back(); + stateTransition(); + return 0; +} + +uint32_t TDenseProtocol::readFieldBegin(string& name, + TType& fieldType, + int16_t& fieldId) { + uint32_t xfer = 0; + + // For optional fields, check to see if they are there. + while (FMT.is_optional) { + bool is_present; + xfer += subReadBool(is_present); + if (is_present) { + break; + } + IDX++; + } + + // Once we hit a mandatory field, or an optional field that is present, + // we know that FMT and FTS point to the appropriate field. + + fieldId = FMT.tag; + fieldType = FTS->ttype; + + // Normally, we push the TypeSpec that we are about to read, + // but no reading is done for T_STOP. + if (FTS->ttype != T_STOP) { + ts_stack_.push_back(FTS); + } + return xfer; +} + +uint32_t TDenseProtocol::readFieldEnd() { + IDX++; + return 0; +} + +uint32_t TDenseProtocol::readMapBegin(TType& keyType, + TType& valType, + uint32_t& size) { + checkTType(T_MAP); + + uint32_t xfer = 0; + int32_t sizei; + xfer += subReadI32(sizei); + if (sizei < 0) { + resetState(); + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (container_limit_ && sizei > container_limit_) { + resetState(); + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + size = (uint32_t)sizei; + + keyType = ST1->ttype; + valType = ST2->ttype; + + ts_stack_.push_back(ST1); + mkv_stack_.push_back(true); + + return xfer; +} + +uint32_t TDenseProtocol::readMapEnd() { + ts_stack_.pop_back(); + mkv_stack_.pop_back(); + stateTransition(); + return 0; +} + +uint32_t TDenseProtocol::readListBegin(TType& elemType, + uint32_t& size) { + checkTType(T_LIST); + + uint32_t xfer = 0; + int32_t sizei; + xfer += subReadI32(sizei); + if (sizei < 0) { + resetState(); + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (container_limit_ && sizei > container_limit_) { + resetState(); + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + size = (uint32_t)sizei; + + elemType = ST1->ttype; + + ts_stack_.push_back(ST1); + + return xfer; +} + +uint32_t TDenseProtocol::readListEnd() { + ts_stack_.pop_back(); + stateTransition(); + return 0; +} + +uint32_t TDenseProtocol::readSetBegin(TType& elemType, + uint32_t& size) { + checkTType(T_SET); + + uint32_t xfer = 0; + int32_t sizei; + xfer += subReadI32(sizei); + if (sizei < 0) { + resetState(); + throw TProtocolException(TProtocolException::NEGATIVE_SIZE); + } else if (container_limit_ && sizei > container_limit_) { + resetState(); + throw TProtocolException(TProtocolException::SIZE_LIMIT); + } + size = (uint32_t)sizei; + + elemType = ST1->ttype; + + ts_stack_.push_back(ST1); + + return xfer; +} + +uint32_t TDenseProtocol::readSetEnd() { + ts_stack_.pop_back(); + stateTransition(); + return 0; +} + +uint32_t TDenseProtocol::readBool(bool& value) { + checkTType(T_BOOL); + stateTransition(); + return TBinaryProtocol::readBool(value); +} + +uint32_t TDenseProtocol::readByte(int8_t& byte) { + checkTType(T_BYTE); + stateTransition(); + return TBinaryProtocol::readByte(byte); +} + +uint32_t TDenseProtocol::readI16(int16_t& i16) { + checkTType(T_I16); + stateTransition(); + uint64_t u64; + uint32_t rv = vlqRead(u64); + int64_t val = (int64_t)u64; + if (UNLIKELY(val > INT16_MAX || val < INT16_MIN)) { + resetState(); + throw TProtocolException(TProtocolException::INVALID_DATA, + "i16 out of range."); + } + i16 = (int16_t)val; + return rv; +} + +uint32_t TDenseProtocol::readI32(int32_t& i32) { + checkTType(T_I32); + stateTransition(); + uint64_t u64; + uint32_t rv = vlqRead(u64); + int64_t val = (int64_t)u64; + if (UNLIKELY(val > INT32_MAX || val < INT32_MIN)) { + resetState(); + throw TProtocolException(TProtocolException::INVALID_DATA, + "i32 out of range."); + } + i32 = (int32_t)val; + return rv; +} + +uint32_t TDenseProtocol::readI64(int64_t& i64) { + checkTType(T_I64); + stateTransition(); + uint64_t u64; + uint32_t rv = vlqRead(u64); + int64_t val = (int64_t)u64; + if (UNLIKELY(val > INT64_MAX || val < INT64_MIN)) { + resetState(); + throw TProtocolException(TProtocolException::INVALID_DATA, + "i64 out of range."); + } + i64 = (int64_t)val; + return rv; +} + +uint32_t TDenseProtocol::readDouble(double& dub) { + checkTType(T_DOUBLE); + stateTransition(); + return TBinaryProtocol::readDouble(dub); +} + +uint32_t TDenseProtocol::readString(std::string& str) { + checkTType(T_STRING); + stateTransition(); + return subReadString(str); +} + +uint32_t TDenseProtocol::readBinary(std::string& str) { + return TDenseProtocol::readString(str); +} + +uint32_t TDenseProtocol::subReadI32(int32_t& i32) { + uint64_t u64; + uint32_t rv = vlqRead(u64); + int64_t val = (int64_t)u64; + if (UNLIKELY(val > INT32_MAX || val < INT32_MIN)) { + resetState(); + throw TProtocolException(TProtocolException::INVALID_DATA, + "i32 out of range."); + } + i32 = (int32_t)val; + return rv; +} + +uint32_t TDenseProtocol::subReadString(std::string& str) { + uint32_t xfer; + int32_t size; + xfer = subReadI32(size); + return xfer + readStringBody(str, size); +} + +}}} // apache::thrift::protocol diff --git a/lib/cpp/src/protocol/TDenseProtocol.h b/lib/cpp/src/protocol/TDenseProtocol.h new file mode 100644 index 00000000..7655a479 --- /dev/null +++ b/lib/cpp/src/protocol/TDenseProtocol.h @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TDENSEPROTOCOL_H_ +#define _THRIFT_PROTOCOL_TDENSEPROTOCOL_H_ 1 + +#include "TBinaryProtocol.h" + +namespace apache { namespace thrift { namespace protocol { + +/** + * !!!WARNING!!! + * This class is still highly experimental. Incompatible changes + * WILL be made to it without notice. DO NOT USE IT YET unless + * you are coordinating your testing with the author. + * + * The dense protocol is designed to use as little space as possible. + * + * There are two types of dense protocol instances. Standalone instances + * are not used for RPC and just encoded and decode structures of + * a predetermined type. Non-standalone instances are used for RPC. + * Currently, only standalone instances exist. + * + * To use a standalone dense protocol object, you must set the type_spec + * property (either in the constructor, or with setTypeSpec) to the local + * reflection TypeSpec of the structures you will write to (or read from) the + * protocol instance. + * + * BEST PRACTICES: + * - Never use optional for primitives or containers. + * - Only use optional for structures if they are very big and very rarely set. + * - All integers are variable-length, so you can use i64 without bloating. + * - NEVER EVER change the struct definitions IN ANY WAY without either + * changing your cache keys or talking to dreiss. + * + * TODO(dreiss): New class write with old meta. + * + * We override all of TBinaryProtocol's methods. + * We inherit so that we can can explicitly call TBPs's primitive-writing + * methods within our versions. + * + */ +class TDenseProtocol : public TBinaryProtocol { + protected: + static const int32_t VERSION_MASK = 0xffff0000; + // VERSION_1 (0x80010000) is taken by TBinaryProtocol. + static const int32_t VERSION_2 = 0x80020000; + + public: + typedef apache::thrift::reflection::local::TypeSpec TypeSpec; + static const int FP_PREFIX_LEN; + + /** + * @param tran The transport to use. + * @param type_spec The TypeSpec of the structures using this protocol. + */ + TDenseProtocol(boost::shared_ptr trans, + TypeSpec* type_spec = NULL) : + TBinaryProtocol(trans), + type_spec_(type_spec), + standalone_(true) + {} + + void setTypeSpec(TypeSpec* type_spec) { + type_spec_ = type_spec; + } + TypeSpec* getTypeSpec() { + return type_spec_; + } + + + /* + * Writing functions. + */ + + virtual uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid); + + virtual uint32_t writeMessageEnd(); + + + virtual uint32_t writeStructBegin(const char* name); + + virtual uint32_t writeStructEnd(); + + virtual uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId); + + virtual uint32_t writeFieldEnd(); + + virtual uint32_t writeFieldStop(); + + virtual uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size); + + virtual uint32_t writeMapEnd(); + + virtual uint32_t writeListBegin(const TType elemType, + const uint32_t size); + + virtual uint32_t writeListEnd(); + + virtual uint32_t writeSetBegin(const TType elemType, + const uint32_t size); + + virtual uint32_t writeSetEnd(); + + virtual uint32_t writeBool(const bool value); + + virtual uint32_t writeByte(const int8_t byte); + + virtual uint32_t writeI16(const int16_t i16); + + virtual uint32_t writeI32(const int32_t i32); + + virtual uint32_t writeI64(const int64_t i64); + + virtual uint32_t writeDouble(const double dub); + + virtual uint32_t writeString(const std::string& str); + + virtual uint32_t writeBinary(const std::string& str); + + + /* + * Helper writing functions (don't do state transitions). + */ + inline uint32_t subWriteI32(const int32_t i32); + + inline uint32_t subWriteString(const std::string& str); + + uint32_t subWriteBool(const bool value) { + return TBinaryProtocol::writeBool(value); + } + + + /* + * Reading functions + */ + + uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid); + + uint32_t readMessageEnd(); + + uint32_t readStructBegin(std::string& name); + + uint32_t readStructEnd(); + + uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId); + + uint32_t readFieldEnd(); + + uint32_t readMapBegin(TType& keyType, + TType& valType, + uint32_t& size); + + uint32_t readMapEnd(); + + uint32_t readListBegin(TType& elemType, + uint32_t& size); + + uint32_t readListEnd(); + + uint32_t readSetBegin(TType& elemType, + uint32_t& size); + + uint32_t readSetEnd(); + + uint32_t readBool(bool& value); + + uint32_t readByte(int8_t& byte); + + uint32_t readI16(int16_t& i16); + + uint32_t readI32(int32_t& i32); + + uint32_t readI64(int64_t& i64); + + uint32_t readDouble(double& dub); + + uint32_t readString(std::string& str); + + uint32_t readBinary(std::string& str); + + /* + * Helper reading functions (don't do state transitions). + */ + inline uint32_t subReadI32(int32_t& i32); + + inline uint32_t subReadString(std::string& str); + + uint32_t subReadBool(bool& value) { + return TBinaryProtocol::readBool(value); + } + + + private: + + // Implementation functions, documented in the .cpp. + inline void checkTType(const TType ttype); + inline void stateTransition(); + + // Read and write variable-length integers. + // Uses the same technique as the MIDI file format. + inline uint32_t vlqRead(uint64_t& vlq); + inline uint32_t vlqWrite(uint64_t vlq); + + // Called before throwing an exception to make the object reusable. + void resetState() { + ts_stack_.clear(); + idx_stack_.clear(); + mkv_stack_.clear(); + } + + // TypeSpec of the top-level structure to write, + // for standalone protocol objects. + TypeSpec* type_spec_; + + std::vector ts_stack_; // TypeSpec stack. + std::vector idx_stack_; // InDeX stack. + std::vector mkv_stack_; // Map Key/Vlue stack. + // True = key, False = value. + + // True iff this is a standalone instance (no RPC). + bool standalone_; +}; + +}}} // apache::thrift::protocol + +#endif // #ifndef _THRIFT_PROTOCOL_TDENSEPROTOCOL_H_ diff --git a/lib/cpp/src/protocol/TJSONProtocol.cpp b/lib/cpp/src/protocol/TJSONProtocol.cpp new file mode 100644 index 00000000..2a9c8f0b --- /dev/null +++ b/lib/cpp/src/protocol/TJSONProtocol.cpp @@ -0,0 +1,998 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "TJSONProtocol.h" + +#include +#include +#include "TBase64Utils.h" +#include + +using namespace apache::thrift::transport; + +namespace apache { namespace thrift { namespace protocol { + + +// Static data + +static const uint8_t kJSONObjectStart = '{'; +static const uint8_t kJSONObjectEnd = '}'; +static const uint8_t kJSONArrayStart = '['; +static const uint8_t kJSONArrayEnd = ']'; +static const uint8_t kJSONNewline = '\n'; +static const uint8_t kJSONPairSeparator = ':'; +static const uint8_t kJSONElemSeparator = ','; +static const uint8_t kJSONBackslash = '\\'; +static const uint8_t kJSONStringDelimiter = '"'; +static const uint8_t kJSONZeroChar = '0'; +static const uint8_t kJSONEscapeChar = 'u'; + +static const std::string kJSONEscapePrefix("\\u00"); + +static const uint32_t kThriftVersion1 = 1; + +static const std::string kThriftNan("NaN"); +static const std::string kThriftInfinity("Infinity"); +static const std::string kThriftNegativeInfinity("-Infinity"); + +static const std::string kTypeNameBool("tf"); +static const std::string kTypeNameByte("i8"); +static const std::string kTypeNameI16("i16"); +static const std::string kTypeNameI32("i32"); +static const std::string kTypeNameI64("i64"); +static const std::string kTypeNameDouble("dbl"); +static const std::string kTypeNameStruct("rec"); +static const std::string kTypeNameString("str"); +static const std::string kTypeNameMap("map"); +static const std::string kTypeNameList("lst"); +static const std::string kTypeNameSet("set"); + +static const std::string &getTypeNameForTypeID(TType typeID) { + switch (typeID) { + case T_BOOL: + return kTypeNameBool; + case T_BYTE: + return kTypeNameByte; + case T_I16: + return kTypeNameI16; + case T_I32: + return kTypeNameI32; + case T_I64: + return kTypeNameI64; + case T_DOUBLE: + return kTypeNameDouble; + case T_STRING: + return kTypeNameString; + case T_STRUCT: + return kTypeNameStruct; + case T_MAP: + return kTypeNameMap; + case T_SET: + return kTypeNameSet; + case T_LIST: + return kTypeNameList; + default: + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "Unrecognized type"); + } +} + +static TType getTypeIDForTypeName(const std::string &name) { + TType result = T_STOP; // Sentinel value + if (name.length() > 1) { + switch (name[0]) { + case 'd': + result = T_DOUBLE; + break; + case 'i': + switch (name[1]) { + case '8': + result = T_BYTE; + break; + case '1': + result = T_I16; + break; + case '3': + result = T_I32; + break; + case '6': + result = T_I64; + break; + } + break; + case 'l': + result = T_LIST; + break; + case 'm': + result = T_MAP; + break; + case 'r': + result = T_STRUCT; + break; + case 's': + if (name[1] == 't') { + result = T_STRING; + } + else if (name[1] == 'e') { + result = T_SET; + } + break; + case 't': + result = T_BOOL; + break; + } + } + if (result == T_STOP) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + "Unrecognized type"); + } + return result; +} + + +// This table describes the handling for the first 0x30 characters +// 0 : escape using "\u00xx" notation +// 1 : just output index +// : escape using "\" notation +static const uint8_t kJSONCharTable[0x30] = { +// 0 1 2 3 4 5 6 7 8 9 A B C D E F + 0, 0, 0, 0, 0, 0, 0, 0,'b','t','n', 0,'f','r', 0, 0, // 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1 + 1, 1,'"', 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2 +}; + + +// This string's characters must match up with the elements in kEscapeCharVals. +// I don't have '/' on this list even though it appears on www.json.org -- +// it is not in the RFC +const static std::string kEscapeChars("\"\\bfnrt"); + +// The elements of this array must match up with the sequence of characters in +// kEscapeChars +const static uint8_t kEscapeCharVals[7] = { + '"', '\\', '\b', '\f', '\n', '\r', '\t', +}; + + +// Static helper functions + +// Read 1 character from the transport trans and verify that it is the +// expected character ch. +// Throw a protocol exception if it is not. +static uint32_t readSyntaxChar(TJSONProtocol::LookaheadReader &reader, + uint8_t ch) { + uint8_t ch2 = reader.read(); + if (ch2 != ch) { + throw TProtocolException(TProtocolException::INVALID_DATA, + "Expected \'" + std::string((char *)&ch, 1) + + "\'; got \'" + std::string((char *)&ch2, 1) + + "\'."); + } + return 1; +} + +// Return the integer value of a hex character ch. +// Throw a protocol exception if the character is not [0-9a-f]. +static uint8_t hexVal(uint8_t ch) { + if ((ch >= '0') && (ch <= '9')) { + return ch - '0'; + } + else if ((ch >= 'a') && (ch <= 'f')) { + return ch - 'a'; + } + else { + throw TProtocolException(TProtocolException::INVALID_DATA, + "Expected hex val ([0-9a-f]); got \'" + + std::string((char *)&ch, 1) + "\'."); + } +} + +// Return the hex character representing the integer val. The value is masked +// to make sure it is in the correct range. +static uint8_t hexChar(uint8_t val) { + val &= 0x0F; + if (val < 10) { + return val + '0'; + } + else { + return val + 'a'; + } +} + +// Return true if the character ch is in [-+0-9.Ee]; false otherwise +static bool isJSONNumeric(uint8_t ch) { + switch (ch) { + case '+': + case '-': + case '.': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 'E': + case 'e': + return true; + } + return false; +} + + +/** + * Class to serve as base JSON context and as base class for other context + * implementations + */ +class TJSONContext { + + public: + + TJSONContext() {}; + + virtual ~TJSONContext() {}; + + /** + * Write context data to the transport. Default is to do nothing. + */ + virtual uint32_t write(TTransport &trans) { + return 0; + }; + + /** + * Read context data from the transport. Default is to do nothing. + */ + virtual uint32_t read(TJSONProtocol::LookaheadReader &reader) { + return 0; + }; + + /** + * Return true if numbers need to be escaped as strings in this context. + * Default behavior is to return false. + */ + virtual bool escapeNum() { + return false; + } +}; + +// Context class for object member key-value pairs +class JSONPairContext : public TJSONContext { + +public: + + JSONPairContext() : + first_(true), + colon_(true) { + } + + uint32_t write(TTransport &trans) { + if (first_) { + first_ = false; + colon_ = true; + return 0; + } + else { + trans.write(colon_ ? &kJSONPairSeparator : &kJSONElemSeparator, 1); + colon_ = !colon_; + return 1; + } + } + + uint32_t read(TJSONProtocol::LookaheadReader &reader) { + if (first_) { + first_ = false; + colon_ = true; + return 0; + } + else { + uint8_t ch = (colon_ ? kJSONPairSeparator : kJSONElemSeparator); + colon_ = !colon_; + return readSyntaxChar(reader, ch); + } + } + + // Numbers must be turned into strings if they are the key part of a pair + virtual bool escapeNum() { + return colon_; + } + + private: + + bool first_; + bool colon_; +}; + +// Context class for lists +class JSONListContext : public TJSONContext { + +public: + + JSONListContext() : + first_(true) { + } + + uint32_t write(TTransport &trans) { + if (first_) { + first_ = false; + return 0; + } + else { + trans.write(&kJSONElemSeparator, 1); + return 1; + } + } + + uint32_t read(TJSONProtocol::LookaheadReader &reader) { + if (first_) { + first_ = false; + return 0; + } + else { + return readSyntaxChar(reader, kJSONElemSeparator); + } + } + + private: + bool first_; +}; + + +TJSONProtocol::TJSONProtocol(boost::shared_ptr ptrans) : + TProtocol(ptrans), + context_(new TJSONContext()), + reader_(*ptrans) { +} + +TJSONProtocol::~TJSONProtocol() {} + +void TJSONProtocol::pushContext(boost::shared_ptr c) { + contexts_.push(context_); + context_ = c; +} + +void TJSONProtocol::popContext() { + context_ = contexts_.top(); + contexts_.pop(); +} + +// Write the character ch as a JSON escape sequence ("\u00xx") +uint32_t TJSONProtocol::writeJSONEscapeChar(uint8_t ch) { + trans_->write((const uint8_t *)kJSONEscapePrefix.c_str(), + kJSONEscapePrefix.length()); + uint8_t outCh = hexChar(ch >> 4); + trans_->write(&outCh, 1); + outCh = hexChar(ch); + trans_->write(&outCh, 1); + return 6; +} + +// Write the character ch as part of a JSON string, escaping as appropriate. +uint32_t TJSONProtocol::writeJSONChar(uint8_t ch) { + if (ch >= 0x30) { + if (ch == kJSONBackslash) { // Only special character >= 0x30 is '\' + trans_->write(&kJSONBackslash, 1); + trans_->write(&kJSONBackslash, 1); + return 2; + } + else { + trans_->write(&ch, 1); + return 1; + } + } + else { + uint8_t outCh = kJSONCharTable[ch]; + // Check if regular character, backslash escaped, or JSON escaped + if (outCh == 1) { + trans_->write(&ch, 1); + return 1; + } + else if (outCh > 1) { + trans_->write(&kJSONBackslash, 1); + trans_->write(&outCh, 1); + return 2; + } + else { + return writeJSONEscapeChar(ch); + } + } +} + +// Write out the contents of the string str as a JSON string, escaping +// characters as appropriate. +uint32_t TJSONProtocol::writeJSONString(const std::string &str) { + uint32_t result = context_->write(*trans_); + result += 2; // For quotes + trans_->write(&kJSONStringDelimiter, 1); + std::string::const_iterator iter(str.begin()); + std::string::const_iterator end(str.end()); + while (iter != end) { + result += writeJSONChar(*iter++); + } + trans_->write(&kJSONStringDelimiter, 1); + return result; +} + +// Write out the contents of the string as JSON string, base64-encoding +// the string's contents, and escaping as appropriate +uint32_t TJSONProtocol::writeJSONBase64(const std::string &str) { + uint32_t result = context_->write(*trans_); + result += 2; // For quotes + trans_->write(&kJSONStringDelimiter, 1); + uint8_t b[4]; + const uint8_t *bytes = (const uint8_t *)str.c_str(); + uint32_t len = str.length(); + while (len >= 3) { + // Encode 3 bytes at a time + base64_encode(bytes, 3, b); + trans_->write(b, 4); + result += 4; + bytes += 3; + len -=3; + } + if (len) { // Handle remainder + base64_encode(bytes, len, b); + trans_->write(b, len + 1); + result += len + 1; + } + trans_->write(&kJSONStringDelimiter, 1); + return result; +} + +// Convert the given integer type to a JSON number, or a string +// if the context requires it (eg: key in a map pair). +template +uint32_t TJSONProtocol::writeJSONInteger(NumberType num) { + uint32_t result = context_->write(*trans_); + std::string val(boost::lexical_cast(num)); + bool escapeNum = context_->escapeNum(); + if (escapeNum) { + trans_->write(&kJSONStringDelimiter, 1); + result += 1; + } + trans_->write((const uint8_t *)val.c_str(), val.length()); + result += val.length(); + if (escapeNum) { + trans_->write(&kJSONStringDelimiter, 1); + result += 1; + } + return result; +} + +// Convert the given double to a JSON string, which is either the number, +// "NaN" or "Infinity" or "-Infinity". +uint32_t TJSONProtocol::writeJSONDouble(double num) { + uint32_t result = context_->write(*trans_); + std::string val(boost::lexical_cast(num)); + + // Normalize output of boost::lexical_cast for NaNs and Infinities + bool special = false; + switch (val[0]) { + case 'N': + case 'n': + val = kThriftNan; + special = true; + break; + case 'I': + case 'i': + val = kThriftInfinity; + special = true; + break; + case '-': + if ((val[1] == 'I') || (val[1] == 'i')) { + val = kThriftNegativeInfinity; + special = true; + } + break; + } + + bool escapeNum = special || context_->escapeNum(); + if (escapeNum) { + trans_->write(&kJSONStringDelimiter, 1); + result += 1; + } + trans_->write((const uint8_t *)val.c_str(), val.length()); + result += val.length(); + if (escapeNum) { + trans_->write(&kJSONStringDelimiter, 1); + result += 1; + } + return result; +} + +uint32_t TJSONProtocol::writeJSONObjectStart() { + uint32_t result = context_->write(*trans_); + trans_->write(&kJSONObjectStart, 1); + pushContext(boost::shared_ptr(new JSONPairContext())); + return result + 1; +} + +uint32_t TJSONProtocol::writeJSONObjectEnd() { + popContext(); + trans_->write(&kJSONObjectEnd, 1); + return 1; +} + +uint32_t TJSONProtocol::writeJSONArrayStart() { + uint32_t result = context_->write(*trans_); + trans_->write(&kJSONArrayStart, 1); + pushContext(boost::shared_ptr(new JSONListContext())); + return result + 1; +} + +uint32_t TJSONProtocol::writeJSONArrayEnd() { + popContext(); + trans_->write(&kJSONArrayEnd, 1); + return 1; +} + +uint32_t TJSONProtocol::writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid) { + uint32_t result = writeJSONArrayStart(); + result += writeJSONInteger(kThriftVersion1); + result += writeJSONString(name); + result += writeJSONInteger(messageType); + result += writeJSONInteger(seqid); + return result; +} + +uint32_t TJSONProtocol::writeMessageEnd() { + return writeJSONArrayEnd(); +} + +uint32_t TJSONProtocol::writeStructBegin(const char* name) { + return writeJSONObjectStart(); +} + +uint32_t TJSONProtocol::writeStructEnd() { + return writeJSONObjectEnd(); +} + +uint32_t TJSONProtocol::writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId) { + uint32_t result = writeJSONInteger(fieldId); + result += writeJSONObjectStart(); + result += writeJSONString(getTypeNameForTypeID(fieldType)); + return result; +} + +uint32_t TJSONProtocol::writeFieldEnd() { + return writeJSONObjectEnd(); +} + +uint32_t TJSONProtocol::writeFieldStop() { + return 0; +} + +uint32_t TJSONProtocol::writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size) { + uint32_t result = writeJSONArrayStart(); + result += writeJSONString(getTypeNameForTypeID(keyType)); + result += writeJSONString(getTypeNameForTypeID(valType)); + result += writeJSONInteger((int64_t)size); + result += writeJSONObjectStart(); + return result; +} + +uint32_t TJSONProtocol::writeMapEnd() { + return writeJSONObjectEnd() + writeJSONArrayEnd(); +} + +uint32_t TJSONProtocol::writeListBegin(const TType elemType, + const uint32_t size) { + uint32_t result = writeJSONArrayStart(); + result += writeJSONString(getTypeNameForTypeID(elemType)); + result += writeJSONInteger((int64_t)size); + return result; +} + +uint32_t TJSONProtocol::writeListEnd() { + return writeJSONArrayEnd(); +} + +uint32_t TJSONProtocol::writeSetBegin(const TType elemType, + const uint32_t size) { + uint32_t result = writeJSONArrayStart(); + result += writeJSONString(getTypeNameForTypeID(elemType)); + result += writeJSONInteger((int64_t)size); + return result; +} + +uint32_t TJSONProtocol::writeSetEnd() { + return writeJSONArrayEnd(); +} + +uint32_t TJSONProtocol::writeBool(const bool value) { + return writeJSONInteger(value); +} + +uint32_t TJSONProtocol::writeByte(const int8_t byte) { + // writeByte() must be handled specially becuase boost::lexical cast sees + // int8_t as a text type instead of an integer type + return writeJSONInteger((int16_t)byte); +} + +uint32_t TJSONProtocol::writeI16(const int16_t i16) { + return writeJSONInteger(i16); +} + +uint32_t TJSONProtocol::writeI32(const int32_t i32) { + return writeJSONInteger(i32); +} + +uint32_t TJSONProtocol::writeI64(const int64_t i64) { + return writeJSONInteger(i64); +} + +uint32_t TJSONProtocol::writeDouble(const double dub) { + return writeJSONDouble(dub); +} + +uint32_t TJSONProtocol::writeString(const std::string& str) { + return writeJSONString(str); +} + +uint32_t TJSONProtocol::writeBinary(const std::string& str) { + return writeJSONBase64(str); +} + + /** + * Reading functions + */ + +// Reads 1 byte and verifies that it matches ch. +uint32_t TJSONProtocol::readJSONSyntaxChar(uint8_t ch) { + return readSyntaxChar(reader_, ch); +} + +// Decodes the four hex parts of a JSON escaped string character and returns +// the character via out. The first two characters must be "00". +uint32_t TJSONProtocol::readJSONEscapeChar(uint8_t *out) { + uint8_t b[2]; + readJSONSyntaxChar(kJSONZeroChar); + readJSONSyntaxChar(kJSONZeroChar); + b[0] = reader_.read(); + b[1] = reader_.read(); + *out = (hexVal(b[0]) << 4) + hexVal(b[1]); + return 4; +} + +// Decodes a JSON string, including unescaping, and returns the string via str +uint32_t TJSONProtocol::readJSONString(std::string &str, bool skipContext) { + uint32_t result = (skipContext ? 0 : context_->read(reader_)); + result += readJSONSyntaxChar(kJSONStringDelimiter); + uint8_t ch; + str.clear(); + while (true) { + ch = reader_.read(); + ++result; + if (ch == kJSONStringDelimiter) { + break; + } + if (ch == kJSONBackslash) { + ch = reader_.read(); + ++result; + if (ch == kJSONEscapeChar) { + result += readJSONEscapeChar(&ch); + } + else { + size_t pos = kEscapeChars.find(ch); + if (pos == std::string::npos) { + throw TProtocolException(TProtocolException::INVALID_DATA, + "Expected control char, got '" + + std::string((const char *)&ch, 1) + "'."); + } + ch = kEscapeCharVals[pos]; + } + } + str += ch; + } + return result; +} + +// Reads a block of base64 characters, decoding it, and returns via str +uint32_t TJSONProtocol::readJSONBase64(std::string &str) { + std::string tmp; + uint32_t result = readJSONString(tmp); + uint8_t *b = (uint8_t *)tmp.c_str(); + uint32_t len = tmp.length(); + str.clear(); + while (len >= 4) { + base64_decode(b, 4); + str.append((const char *)b, 3); + b += 4; + len -= 4; + } + // Don't decode if we hit the end or got a single leftover byte (invalid + // base64 but legal for skip of regular string type) + if (len > 1) { + base64_decode(b, len); + str.append((const char *)b, len - 1); + } + return result; +} + +// Reads a sequence of characters, stopping at the first one that is not +// a valid JSON numeric character. +uint32_t TJSONProtocol::readJSONNumericChars(std::string &str) { + uint32_t result = 0; + str.clear(); + while (true) { + uint8_t ch = reader_.peek(); + if (!isJSONNumeric(ch)) { + break; + } + reader_.read(); + str += ch; + ++result; + } + return result; +} + +// Reads a sequence of characters and assembles them into a number, +// returning them via num +template +uint32_t TJSONProtocol::readJSONInteger(NumberType &num) { + uint32_t result = context_->read(reader_); + if (context_->escapeNum()) { + result += readJSONSyntaxChar(kJSONStringDelimiter); + } + std::string str; + result += readJSONNumericChars(str); + try { + num = boost::lexical_cast(str); + } + catch (boost::bad_lexical_cast e) { + throw new TProtocolException(TProtocolException::INVALID_DATA, + "Expected numeric value; got \"" + str + + "\""); + } + if (context_->escapeNum()) { + result += readJSONSyntaxChar(kJSONStringDelimiter); + } + return result; +} + +// Reads a JSON number or string and interprets it as a double. +uint32_t TJSONProtocol::readJSONDouble(double &num) { + uint32_t result = context_->read(reader_); + std::string str; + if (reader_.peek() == kJSONStringDelimiter) { + result += readJSONString(str, true); + // Check for NaN, Infinity and -Infinity + if (str == kThriftNan) { + num = HUGE_VAL/HUGE_VAL; // generates NaN + } + else if (str == kThriftInfinity) { + num = HUGE_VAL; + } + else if (str == kThriftNegativeInfinity) { + num = -HUGE_VAL; + } + else { + if (!context_->escapeNum()) { + // Throw exception -- we should not be in a string in this case + throw new TProtocolException(TProtocolException::INVALID_DATA, + "Numeric data unexpectedly quoted"); + } + try { + num = boost::lexical_cast(str); + } + catch (boost::bad_lexical_cast e) { + throw new TProtocolException(TProtocolException::INVALID_DATA, + "Expected numeric value; got \"" + str + + "\""); + } + } + } + else { + if (context_->escapeNum()) { + // This will throw - we should have had a quote if escapeNum == true + readJSONSyntaxChar(kJSONStringDelimiter); + } + result += readJSONNumericChars(str); + try { + num = boost::lexical_cast(str); + } + catch (boost::bad_lexical_cast e) { + throw new TProtocolException(TProtocolException::INVALID_DATA, + "Expected numeric value; got \"" + str + + "\""); + } + } + return result; +} + +uint32_t TJSONProtocol::readJSONObjectStart() { + uint32_t result = context_->read(reader_); + result += readJSONSyntaxChar(kJSONObjectStart); + pushContext(boost::shared_ptr(new JSONPairContext())); + return result; +} + +uint32_t TJSONProtocol::readJSONObjectEnd() { + uint32_t result = readJSONSyntaxChar(kJSONObjectEnd); + popContext(); + return result; +} + +uint32_t TJSONProtocol::readJSONArrayStart() { + uint32_t result = context_->read(reader_); + result += readJSONSyntaxChar(kJSONArrayStart); + pushContext(boost::shared_ptr(new JSONListContext())); + return result; +} + +uint32_t TJSONProtocol::readJSONArrayEnd() { + uint32_t result = readJSONSyntaxChar(kJSONArrayEnd); + popContext(); + return result; +} + +uint32_t TJSONProtocol::readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid) { + uint32_t result = readJSONArrayStart(); + uint64_t tmpVal = 0; + result += readJSONInteger(tmpVal); + if (tmpVal != kThriftVersion1) { + throw TProtocolException(TProtocolException::BAD_VERSION, + "Message contained bad version."); + } + result += readJSONString(name); + result += readJSONInteger(tmpVal); + messageType = (TMessageType)tmpVal; + result += readJSONInteger(tmpVal); + seqid = tmpVal; + return result; +} + +uint32_t TJSONProtocol::readMessageEnd() { + return readJSONArrayEnd(); +} + +uint32_t TJSONProtocol::readStructBegin(std::string& name) { + return readJSONObjectStart(); +} + +uint32_t TJSONProtocol::readStructEnd() { + return readJSONObjectEnd(); +} + +uint32_t TJSONProtocol::readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId) { + uint32_t result = 0; + // Check if we hit the end of the list + uint8_t ch = reader_.peek(); + if (ch == kJSONObjectEnd) { + fieldType = apache::thrift::protocol::T_STOP; + } + else { + uint64_t tmpVal = 0; + std::string tmpStr; + result += readJSONInteger(tmpVal); + fieldId = tmpVal; + result += readJSONObjectStart(); + result += readJSONString(tmpStr); + fieldType = getTypeIDForTypeName(tmpStr); + } + return result; +} + +uint32_t TJSONProtocol::readFieldEnd() { + return readJSONObjectEnd(); +} + +uint32_t TJSONProtocol::readMapBegin(TType& keyType, + TType& valType, + uint32_t& size) { + uint64_t tmpVal = 0; + std::string tmpStr; + uint32_t result = readJSONArrayStart(); + result += readJSONString(tmpStr); + keyType = getTypeIDForTypeName(tmpStr); + result += readJSONString(tmpStr); + valType = getTypeIDForTypeName(tmpStr); + result += readJSONInteger(tmpVal); + size = tmpVal; + result += readJSONObjectStart(); + return result; +} + +uint32_t TJSONProtocol::readMapEnd() { + return readJSONObjectEnd() + readJSONArrayEnd(); +} + +uint32_t TJSONProtocol::readListBegin(TType& elemType, + uint32_t& size) { + uint64_t tmpVal = 0; + std::string tmpStr; + uint32_t result = readJSONArrayStart(); + result += readJSONString(tmpStr); + elemType = getTypeIDForTypeName(tmpStr); + result += readJSONInteger(tmpVal); + size = tmpVal; + return result; +} + +uint32_t TJSONProtocol::readListEnd() { + return readJSONArrayEnd(); +} + +uint32_t TJSONProtocol::readSetBegin(TType& elemType, + uint32_t& size) { + uint64_t tmpVal = 0; + std::string tmpStr; + uint32_t result = readJSONArrayStart(); + result += readJSONString(tmpStr); + elemType = getTypeIDForTypeName(tmpStr); + result += readJSONInteger(tmpVal); + size = tmpVal; + return result; +} + +uint32_t TJSONProtocol::readSetEnd() { + return readJSONArrayEnd(); +} + +uint32_t TJSONProtocol::readBool(bool& value) { + return readJSONInteger(value); +} + +// readByte() must be handled properly becuase boost::lexical cast sees int8_t +// as a text type instead of an integer type +uint32_t TJSONProtocol::readByte(int8_t& byte) { + int16_t tmp = (int16_t) byte; + uint32_t result = readJSONInteger(tmp); + assert(tmp < 256); + byte = (int8_t)tmp; + return result; +} + +uint32_t TJSONProtocol::readI16(int16_t& i16) { + return readJSONInteger(i16); +} + +uint32_t TJSONProtocol::readI32(int32_t& i32) { + return readJSONInteger(i32); +} + +uint32_t TJSONProtocol::readI64(int64_t& i64) { + return readJSONInteger(i64); +} + +uint32_t TJSONProtocol::readDouble(double& dub) { + return readJSONDouble(dub); +} + +uint32_t TJSONProtocol::readString(std::string &str) { + return readJSONString(str); +} + +uint32_t TJSONProtocol::readBinary(std::string &str) { + return readJSONBase64(str); +} + +}}} // apache::thrift::protocol diff --git a/lib/cpp/src/protocol/TJSONProtocol.h b/lib/cpp/src/protocol/TJSONProtocol.h new file mode 100644 index 00000000..2df499ac --- /dev/null +++ b/lib/cpp/src/protocol/TJSONProtocol.h @@ -0,0 +1,340 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TJSONPROTOCOL_H_ +#define _THRIFT_PROTOCOL_TJSONPROTOCOL_H_ 1 + +#include "TProtocol.h" + +#include + +namespace apache { namespace thrift { namespace protocol { + +// Forward declaration +class TJSONContext; + +/** + * JSON protocol for Thrift. + * + * Implements a protocol which uses JSON as the wire-format. + * + * Thrift types are represented as described below: + * + * 1. Every Thrift integer type is represented as a JSON number. + * + * 2. Thrift doubles are represented as JSON numbers. Some special values are + * represented as strings: + * a. "NaN" for not-a-number values + * b. "Infinity" for postive infinity + * c. "-Infinity" for negative infinity + * + * 3. Thrift string values are emitted as JSON strings, with appropriate + * escaping. + * + * 4. Thrift binary values are encoded into Base64 and emitted as JSON strings. + * The readBinary() method is written such that it will properly skip if + * called on a Thrift string (although it will decode garbage data). + * + * 5. Thrift structs are represented as JSON objects, with the field ID as the + * key, and the field value represented as a JSON object with a single + * key-value pair. The key is a short string identifier for that type, + * followed by the value. The valid type identifiers are: "tf" for bool, + * "i8" for byte, "i16" for 16-bit integer, "i32" for 32-bit integer, "i64" + * for 64-bit integer, "dbl" for double-precision loating point, "str" for + * string (including binary), "rec" for struct ("records"), "map" for map, + * "lst" for list, "set" for set. + * + * 6. Thrift lists and sets are represented as JSON arrays, with the first + * element of the JSON array being the string identifier for the Thrift + * element type and the second element of the JSON array being the count of + * the Thrift elements. The Thrift elements then follow. + * + * 7. Thrift maps are represented as JSON arrays, with the first two elements + * of the JSON array being the string identifiers for the Thrift key type + * and value type, followed by the count of the Thrift pairs, followed by a + * JSON object containing the key-value pairs. Note that JSON keys can only + * be strings, which means that the key type of the Thrift map should be + * restricted to numeric or string types -- in the case of numerics, they + * are serialized as strings. + * + * 8. Thrift messages are represented as JSON arrays, with the protocol + * version #, the message name, the message type, and the sequence ID as + * the first 4 elements. + * + * More discussion of the double handling is probably warranted. The aim of + * the current implementation is to match as closely as possible the behavior + * of Java's Double.toString(), which has no precision loss. Implementors in + * other languages should strive to achieve that where possible. I have not + * yet verified whether boost:lexical_cast, which is doing that work for me in + * C++, loses any precision, but I am leaving this as a future improvement. I + * may try to provide a C component for this, so that other languages could + * bind to the same underlying implementation for maximum consistency. + * + * Note further that JavaScript itself is not capable of representing + * floating point infinities -- presumably when we have a JavaScript Thrift + * client, this would mean that infinities get converted to not-a-number in + * transmission. I don't know of any work-around for this issue. + * + */ +class TJSONProtocol : public TProtocol { + public: + + TJSONProtocol(boost::shared_ptr ptrans); + + ~TJSONProtocol(); + + private: + + void pushContext(boost::shared_ptr c); + + void popContext(); + + uint32_t writeJSONEscapeChar(uint8_t ch); + + uint32_t writeJSONChar(uint8_t ch); + + uint32_t writeJSONString(const std::string &str); + + uint32_t writeJSONBase64(const std::string &str); + + template + uint32_t writeJSONInteger(NumberType num); + + uint32_t writeJSONDouble(double num); + + uint32_t writeJSONObjectStart() ; + + uint32_t writeJSONObjectEnd(); + + uint32_t writeJSONArrayStart(); + + uint32_t writeJSONArrayEnd(); + + uint32_t readJSONSyntaxChar(uint8_t ch); + + uint32_t readJSONEscapeChar(uint8_t *out); + + uint32_t readJSONString(std::string &str, bool skipContext = false); + + uint32_t readJSONBase64(std::string &str); + + uint32_t readJSONNumericChars(std::string &str); + + template + uint32_t readJSONInteger(NumberType &num); + + uint32_t readJSONDouble(double &num); + + uint32_t readJSONObjectStart(); + + uint32_t readJSONObjectEnd(); + + uint32_t readJSONArrayStart(); + + uint32_t readJSONArrayEnd(); + + public: + + /** + * Writing functions. + */ + + uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid); + + uint32_t writeMessageEnd(); + + uint32_t writeStructBegin(const char* name); + + uint32_t writeStructEnd(); + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId); + + uint32_t writeFieldEnd(); + + uint32_t writeFieldStop(); + + uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size); + + uint32_t writeMapEnd(); + + uint32_t writeListBegin(const TType elemType, + const uint32_t size); + + uint32_t writeListEnd(); + + uint32_t writeSetBegin(const TType elemType, + const uint32_t size); + + uint32_t writeSetEnd(); + + uint32_t writeBool(const bool value); + + uint32_t writeByte(const int8_t byte); + + uint32_t writeI16(const int16_t i16); + + uint32_t writeI32(const int32_t i32); + + uint32_t writeI64(const int64_t i64); + + uint32_t writeDouble(const double dub); + + uint32_t writeString(const std::string& str); + + uint32_t writeBinary(const std::string& str); + + /** + * Reading functions + */ + + uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid); + + uint32_t readMessageEnd(); + + uint32_t readStructBegin(std::string& name); + + uint32_t readStructEnd(); + + uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId); + + uint32_t readFieldEnd(); + + uint32_t readMapBegin(TType& keyType, + TType& valType, + uint32_t& size); + + uint32_t readMapEnd(); + + uint32_t readListBegin(TType& elemType, + uint32_t& size); + + uint32_t readListEnd(); + + uint32_t readSetBegin(TType& elemType, + uint32_t& size); + + uint32_t readSetEnd(); + + uint32_t readBool(bool& value); + + uint32_t readByte(int8_t& byte); + + uint32_t readI16(int16_t& i16); + + uint32_t readI32(int32_t& i32); + + uint32_t readI64(int64_t& i64); + + uint32_t readDouble(double& dub); + + uint32_t readString(std::string& str); + + uint32_t readBinary(std::string& str); + + class LookaheadReader { + + public: + + LookaheadReader(TTransport &trans) : + trans_(&trans), + hasData_(false) { + } + + uint8_t read() { + if (hasData_) { + hasData_ = false; + } + else { + trans_->readAll(&data_, 1); + } + return data_; + } + + uint8_t peek() { + if (!hasData_) { + trans_->readAll(&data_, 1); + } + hasData_ = true; + return data_; + } + + private: + TTransport *trans_; + bool hasData_; + uint8_t data_; + }; + + private: + + std::stack > contexts_; + boost::shared_ptr context_; + LookaheadReader reader_; +}; + +/** + * Constructs input and output protocol objects given transports. + */ +class TJSONProtocolFactory : public TProtocolFactory { + public: + TJSONProtocolFactory() {} + + virtual ~TJSONProtocolFactory() {} + + boost::shared_ptr getProtocol(boost::shared_ptr trans) { + return boost::shared_ptr(new TJSONProtocol(trans)); + } +}; + +}}} // apache::thrift::protocol + + +// TODO(dreiss): Move part of ThriftJSONString into a .cpp file and remove this. +#include + +namespace apache { namespace thrift { + +template + std::string ThriftJSONString(const ThriftStruct& ts) { + using namespace apache::thrift::transport; + using namespace apache::thrift::protocol; + TMemoryBuffer* buffer = new TMemoryBuffer; + boost::shared_ptr trans(buffer); + TJSONProtocol protocol(trans); + + ts.write(&protocol); + + uint8_t* buf; + uint32_t size; + buffer->getBuffer(&buf, &size); + return std::string((char*)buf, (unsigned int)size); +} + +}} // apache::thrift + +#endif // #define _THRIFT_PROTOCOL_TJSONPROTOCOL_H_ 1 diff --git a/lib/cpp/src/protocol/TOneWayProtocol.h b/lib/cpp/src/protocol/TOneWayProtocol.h new file mode 100644 index 00000000..6f08fe1d --- /dev/null +++ b/lib/cpp/src/protocol/TOneWayProtocol.h @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TONEWAYPROTOCOL_H_ +#define _THRIFT_PROTOCOL_TONEWAYPROTOCOL_H_ 1 + +#include "TProtocol.h" + +namespace apache { namespace thrift { namespace protocol { + +/** + * Abstract class for implementing a protocol that can only be written, + * not read. + * + */ +class TWriteOnlyProtocol : public TProtocol { + public: + /** + * @param subclass_name The name of the concrete subclass. + */ + TWriteOnlyProtocol(boost::shared_ptr trans, + const std::string& subclass_name) + : TProtocol(trans) + , subclass_(subclass_name) + {} + + // All writing functions remain abstract. + + /** + * Reading functions all throw an exception. + */ + + uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readMessageEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readStructBegin(std::string& name) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readStructEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readFieldEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readMapBegin(TType& keyType, + TType& valType, + uint32_t& size) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readMapEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readListBegin(TType& elemType, + uint32_t& size) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readListEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readSetBegin(TType& elemType, + uint32_t& size) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readSetEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readBool(bool& value) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readByte(int8_t& byte) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readI16(int16_t& i16) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readI32(int32_t& i32) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readI64(int64_t& i64) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readDouble(double& dub) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readString(std::string& str) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + uint32_t readBinary(std::string& str) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support reading (yet)."); + } + + private: + std::string subclass_; +}; + + +/** + * Abstract class for implementing a protocol that can only be read, + * not written. + * + */ +class TReadOnlyProtocol : public TProtocol { + public: + /** + * @param subclass_name The name of the concrete subclass. + */ + TReadOnlyProtocol(boost::shared_ptr trans, + const std::string& subclass_name) + : TProtocol(trans) + , subclass_(subclass_name) + {} + + // All reading functions remain abstract. + + /** + * Writing functions all throw an exception. + */ + + uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeMessageEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + + uint32_t writeStructBegin(const char* name) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeStructEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeFieldEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeFieldStop() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeMapEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeListBegin(const TType elemType, + const uint32_t size) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeListEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeSetBegin(const TType elemType, + const uint32_t size) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeSetEnd() { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeBool(const bool value) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeByte(const int8_t byte) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeI16(const int16_t i16) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeI32(const int32_t i32) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeI64(const int64_t i64) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeDouble(const double dub) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeString(const std::string& str) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + uint32_t writeBinary(const std::string& str) { + throw TProtocolException(TProtocolException::NOT_IMPLEMENTED, + subclass_ + " does not support writing (yet)."); + } + + private: + std::string subclass_; +}; + +}}} // apache::thrift::protocol + +#endif // #ifndef _THRIFT_PROTOCOL_TBINARYPROTOCOL_H_ diff --git a/lib/cpp/src/protocol/TProtocol.h b/lib/cpp/src/protocol/TProtocol.h new file mode 100644 index 00000000..40258277 --- /dev/null +++ b/lib/cpp/src/protocol/TProtocol.h @@ -0,0 +1,438 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TPROTOCOL_H_ +#define _THRIFT_PROTOCOL_TPROTOCOL_H_ 1 + +#include +#include + +#include +#include + +#include +#include +#include +#include + + +// Use this to get around strict aliasing rules. +// For example, uint64_t i = bitwise_cast(returns_double()); +// The most obvious implementation is to just cast a pointer, +// but that doesn't work. +// For a pretty in-depth explanation of the problem, see +// http://www.cellperformance.com/mike_acton/2006/06/ (...) +// understanding_strict_aliasing.html +template +static inline To bitwise_cast(From from) { + BOOST_STATIC_ASSERT(sizeof(From) == sizeof(To)); + + // BAD!!! These are all broken with -O2. + //return *reinterpret_cast(&from); // BAD!!! + //return *static_cast(static_cast(&from)); // BAD!!! + //return *(To*)(void*)&from; // BAD!!! + + // Super clean and paritally blessed by section 3.9 of the standard. + //unsigned char c[sizeof(from)]; + //memcpy(c, &from, sizeof(from)); + //To to; + //memcpy(&to, c, sizeof(c)); + //return to; + + // Slightly more questionable. + // Same code emitted by GCC. + //To to; + //memcpy(&to, &from, sizeof(from)); + //return to; + + // Technically undefined, but almost universally supported, + // and the most efficient implementation. + union { + From f; + To t; + } u; + u.f = from; + return u.t; +} + + +namespace apache { namespace thrift { namespace protocol { + +using apache::thrift::transport::TTransport; + +#ifdef HAVE_ENDIAN_H +#include +#endif + +#ifndef __BYTE_ORDER +# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN) +# define __BYTE_ORDER BYTE_ORDER +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __BIG_ENDIAN BIG_ENDIAN +# else +# error "Cannot determine endianness" +# endif +#endif + +#if __BYTE_ORDER == __BIG_ENDIAN +# define ntohll(n) (n) +# define htonll(n) (n) +# if defined(__GNUC__) && defined(__GLIBC__) +# include +# define htolell(n) bswap_64(n) +# define letohll(n) bswap_64(n) +# else /* GNUC & GLIBC */ +# define bswap_64(n) \ + ( (((n) & 0xff00000000000000ull) >> 56) \ + | (((n) & 0x00ff000000000000ull) >> 40) \ + | (((n) & 0x0000ff0000000000ull) >> 24) \ + | (((n) & 0x000000ff00000000ull) >> 8) \ + | (((n) & 0x00000000ff000000ull) << 8) \ + | (((n) & 0x0000000000ff0000ull) << 24) \ + | (((n) & 0x000000000000ff00ull) << 40) \ + | (((n) & 0x00000000000000ffull) << 56) ) +# define ntolell(n) bswap_64(n) +# define letonll(n) bswap_64(n) +# endif /* GNUC & GLIBC */ +#elif __BYTE_ORDER == __LITTLE_ENDIAN +# define htolell(n) (n) +# define letohll(n) (n) +# if defined(__GNUC__) && defined(__GLIBC__) +# include +# define ntohll(n) bswap_64(n) +# define htonll(n) bswap_64(n) +# else /* GNUC & GLIBC */ +# define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) ) +# define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) ) +# endif /* GNUC & GLIBC */ +#else /* __BYTE_ORDER */ +# error "Can't define htonll or ntohll!" +#endif + +/** + * Enumerated definition of the types that the Thrift protocol supports. + * Take special note of the T_END type which is used specifically to mark + * the end of a sequence of fields. + */ +enum TType { + T_STOP = 0, + T_VOID = 1, + T_BOOL = 2, + T_BYTE = 3, + T_I08 = 3, + T_I16 = 6, + T_I32 = 8, + T_U64 = 9, + T_I64 = 10, + T_DOUBLE = 4, + T_STRING = 11, + T_UTF7 = 11, + T_STRUCT = 12, + T_MAP = 13, + T_SET = 14, + T_LIST = 15, + T_UTF8 = 16, + T_UTF16 = 17 +}; + +/** + * Enumerated definition of the message types that the Thrift protocol + * supports. + */ +enum TMessageType { + T_CALL = 1, + T_REPLY = 2, + T_EXCEPTION = 3, + T_ONEWAY = 4 +}; + +/** + * Abstract class for a thrift protocol driver. These are all the methods that + * a protocol must implement. Essentially, there must be some way of reading + * and writing all the base types, plus a mechanism for writing out structs + * with indexed fields. + * + * TProtocol objects should not be shared across multiple encoding contexts, + * as they may need to maintain internal state in some protocols (i.e. XML). + * Note that is is acceptable for the TProtocol module to do its own internal + * buffered reads/writes to the underlying TTransport where appropriate (i.e. + * when parsing an input XML stream, reading should be batched rather than + * looking ahead character by character for a close tag). + * + */ +class TProtocol { + public: + virtual ~TProtocol() {} + + /** + * Writing functions. + */ + + virtual uint32_t writeMessageBegin(const std::string& name, + const TMessageType messageType, + const int32_t seqid) = 0; + + virtual uint32_t writeMessageEnd() = 0; + + + virtual uint32_t writeStructBegin(const char* name) = 0; + + virtual uint32_t writeStructEnd() = 0; + + virtual uint32_t writeFieldBegin(const char* name, + const TType fieldType, + const int16_t fieldId) = 0; + + virtual uint32_t writeFieldEnd() = 0; + + virtual uint32_t writeFieldStop() = 0; + + virtual uint32_t writeMapBegin(const TType keyType, + const TType valType, + const uint32_t size) = 0; + + virtual uint32_t writeMapEnd() = 0; + + virtual uint32_t writeListBegin(const TType elemType, + const uint32_t size) = 0; + + virtual uint32_t writeListEnd() = 0; + + virtual uint32_t writeSetBegin(const TType elemType, + const uint32_t size) = 0; + + virtual uint32_t writeSetEnd() = 0; + + virtual uint32_t writeBool(const bool value) = 0; + + virtual uint32_t writeByte(const int8_t byte) = 0; + + virtual uint32_t writeI16(const int16_t i16) = 0; + + virtual uint32_t writeI32(const int32_t i32) = 0; + + virtual uint32_t writeI64(const int64_t i64) = 0; + + virtual uint32_t writeDouble(const double dub) = 0; + + virtual uint32_t writeString(const std::string& str) = 0; + + virtual uint32_t writeBinary(const std::string& str) = 0; + + /** + * Reading functions + */ + + virtual uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid) = 0; + + virtual uint32_t readMessageEnd() = 0; + + virtual uint32_t readStructBegin(std::string& name) = 0; + + virtual uint32_t readStructEnd() = 0; + + virtual uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId) = 0; + + virtual uint32_t readFieldEnd() = 0; + + virtual uint32_t readMapBegin(TType& keyType, + TType& valType, + uint32_t& size) = 0; + + virtual uint32_t readMapEnd() = 0; + + virtual uint32_t readListBegin(TType& elemType, + uint32_t& size) = 0; + + virtual uint32_t readListEnd() = 0; + + virtual uint32_t readSetBegin(TType& elemType, + uint32_t& size) = 0; + + virtual uint32_t readSetEnd() = 0; + + virtual uint32_t readBool(bool& value) = 0; + + virtual uint32_t readByte(int8_t& byte) = 0; + + virtual uint32_t readI16(int16_t& i16) = 0; + + virtual uint32_t readI32(int32_t& i32) = 0; + + virtual uint32_t readI64(int64_t& i64) = 0; + + virtual uint32_t readDouble(double& dub) = 0; + + virtual uint32_t readString(std::string& str) = 0; + + virtual uint32_t readBinary(std::string& str) = 0; + + uint32_t readBool(std::vector::reference ref) { + bool value; + uint32_t rv = readBool(value); + ref = value; + return rv; + } + + /** + * Method to arbitrarily skip over data. + */ + uint32_t skip(TType type) { + switch (type) { + case T_BOOL: + { + bool boolv; + return readBool(boolv); + } + case T_BYTE: + { + int8_t bytev; + return readByte(bytev); + } + case T_I16: + { + int16_t i16; + return readI16(i16); + } + case T_I32: + { + int32_t i32; + return readI32(i32); + } + case T_I64: + { + int64_t i64; + return readI64(i64); + } + case T_DOUBLE: + { + double dub; + return readDouble(dub); + } + case T_STRING: + { + std::string str; + return readBinary(str); + } + case T_STRUCT: + { + uint32_t result = 0; + std::string name; + int16_t fid; + TType ftype; + result += readStructBegin(name); + while (true) { + result += readFieldBegin(name, ftype, fid); + if (ftype == T_STOP) { + break; + } + result += skip(ftype); + result += readFieldEnd(); + } + result += readStructEnd(); + return result; + } + case T_MAP: + { + uint32_t result = 0; + TType keyType; + TType valType; + uint32_t i, size; + result += readMapBegin(keyType, valType, size); + for (i = 0; i < size; i++) { + result += skip(keyType); + result += skip(valType); + } + result += readMapEnd(); + return result; + } + case T_SET: + { + uint32_t result = 0; + TType elemType; + uint32_t i, size; + result += readSetBegin(elemType, size); + for (i = 0; i < size; i++) { + result += skip(elemType); + } + result += readSetEnd(); + return result; + } + case T_LIST: + { + uint32_t result = 0; + TType elemType; + uint32_t i, size; + result += readListBegin(elemType, size); + for (i = 0; i < size; i++) { + result += skip(elemType); + } + result += readListEnd(); + return result; + } + default: + return 0; + } + } + + inline boost::shared_ptr getTransport() { + return ptrans_; + } + + // TODO: remove these two calls, they are for backwards + // compatibility + inline boost::shared_ptr getInputTransport() { + return ptrans_; + } + inline boost::shared_ptr getOutputTransport() { + return ptrans_; + } + + protected: + TProtocol(boost::shared_ptr ptrans): + ptrans_(ptrans) { + trans_ = ptrans.get(); + } + + boost::shared_ptr ptrans_; + TTransport* trans_; + + private: + TProtocol() {} +}; + +/** + * Constructs input and output protocol objects given transports. + */ +class TProtocolFactory { + public: + TProtocolFactory() {} + + virtual ~TProtocolFactory() {} + + virtual boost::shared_ptr getProtocol(boost::shared_ptr trans) = 0; +}; + +}}} // apache::thrift::protocol + +#endif // #define _THRIFT_PROTOCOL_TPROTOCOL_H_ 1 diff --git a/lib/cpp/src/protocol/TProtocolException.h b/lib/cpp/src/protocol/TProtocolException.h new file mode 100644 index 00000000..33011b37 --- /dev/null +++ b/lib/cpp/src/protocol/TProtocolException.h @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_ +#define _THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_ 1 + +#include + +namespace apache { namespace thrift { namespace protocol { + +/** + * Class to encapsulate all the possible types of protocol errors that may + * occur in various protocol systems. This provides a sort of generic + * wrapper around the shitty UNIX E_ error codes that lets a common code + * base of error handling to be used for various types of protocols, i.e. + * pipes etc. + * + */ +class TProtocolException : public apache::thrift::TException { + public: + + /** + * Error codes for the various types of exceptions. + */ + enum TProtocolExceptionType + { UNKNOWN = 0 + , INVALID_DATA = 1 + , NEGATIVE_SIZE = 2 + , SIZE_LIMIT = 3 + , BAD_VERSION = 4 + , NOT_IMPLEMENTED = 5 + }; + + TProtocolException() : + apache::thrift::TException(), + type_(UNKNOWN) {} + + TProtocolException(TProtocolExceptionType type) : + apache::thrift::TException(), + type_(type) {} + + TProtocolException(const std::string& message) : + apache::thrift::TException(message), + type_(UNKNOWN) {} + + TProtocolException(TProtocolExceptionType type, const std::string& message) : + apache::thrift::TException(message), + type_(type) {} + + virtual ~TProtocolException() throw() {} + + /** + * Returns an error code that provides information about the type of error + * that has occurred. + * + * @return Error code + */ + TProtocolExceptionType getType() { + return type_; + } + + virtual const char* what() const throw() { + if (message_.empty()) { + switch (type_) { + case UNKNOWN : return "TProtocolException: Unknown protocol exception"; + case INVALID_DATA : return "TProtocolException: Invalid data"; + case NEGATIVE_SIZE : return "TProtocolException: Negative size"; + case SIZE_LIMIT : return "TProtocolException: Exceeded size limit"; + case BAD_VERSION : return "TProtocolException: Invalid version"; + case NOT_IMPLEMENTED : return "TProtocolException: Not implemented"; + default : return "TProtocolException: (Invalid exception type)"; + } + } else { + return message_.c_str(); + } + } + + protected: + /** + * Error code + */ + TProtocolExceptionType type_; + +}; + +}}} // apache::thrift::protocol + +#endif // #ifndef _THRIFT_PROTOCOL_TPROTOCOLEXCEPTION_H_ diff --git a/lib/cpp/src/protocol/TProtocolTap.h b/lib/cpp/src/protocol/TProtocolTap.h new file mode 100644 index 00000000..5580216a --- /dev/null +++ b/lib/cpp/src/protocol/TProtocolTap.h @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_PROTOCOL_TPROTOCOLTAP_H_ +#define _THRIFT_PROTOCOL_TPROTOCOLTAP_H_ 1 + +#include + +namespace apache { namespace thrift { namespace protocol { + +using apache::thrift::transport::TTransport; + +/** + * Puts a wiretap on a protocol object. Any reads to this class are passed + * through to an enclosed protocol object, but also mirrored as write to a + * second protocol object. + * + */ +class TProtocolTap : public TReadOnlyProtocol { + public: + TProtocolTap(boost::shared_ptr source, + boost::shared_ptr sink) + : TReadOnlyProtocol(source->getTransport(), "TProtocolTap") + , source_(source) + , sink_(sink) + {} + + virtual uint32_t readMessageBegin(std::string& name, + TMessageType& messageType, + int32_t& seqid) { + uint32_t rv = source_->readMessageBegin(name, messageType, seqid); + sink_->writeMessageBegin(name, messageType, seqid); + return rv; + } + + virtual uint32_t readMessageEnd() { + uint32_t rv = source_->readMessageEnd(); + sink_->writeMessageEnd(); + return rv; + } + + virtual uint32_t readStructBegin(std::string& name) { + uint32_t rv = source_->readStructBegin(name); + sink_->writeStructBegin(name.c_str()); + return rv; + } + + virtual uint32_t readStructEnd() { + uint32_t rv = source_->readStructEnd(); + sink_->writeStructEnd(); + return rv; + } + + virtual uint32_t readFieldBegin(std::string& name, + TType& fieldType, + int16_t& fieldId) { + uint32_t rv = source_->readFieldBegin(name, fieldType, fieldId); + if (fieldType == T_STOP) { + sink_->writeFieldStop(); + } else { + sink_->writeFieldBegin(name.c_str(), fieldType, fieldId); + } + return rv; + } + + + virtual uint32_t readFieldEnd() { + uint32_t rv = source_->readFieldEnd(); + sink_->writeFieldEnd(); + return rv; + } + + virtual uint32_t readMapBegin(TType& keyType, + TType& valType, + uint32_t& size) { + uint32_t rv = source_->readMapBegin(keyType, valType, size); + sink_->writeMapBegin(keyType, valType, size); + return rv; + } + + + virtual uint32_t readMapEnd() { + uint32_t rv = source_->readMapEnd(); + sink_->writeMapEnd(); + return rv; + } + + virtual uint32_t readListBegin(TType& elemType, + uint32_t& size) { + uint32_t rv = source_->readListBegin(elemType, size); + sink_->writeListBegin(elemType, size); + return rv; + } + + + virtual uint32_t readListEnd() { + uint32_t rv = source_->readListEnd(); + sink_->writeListEnd(); + return rv; + } + + virtual uint32_t readSetBegin(TType& elemType, + uint32_t& size) { + uint32_t rv = source_->readSetBegin(elemType, size); + sink_->writeSetBegin(elemType, size); + return rv; + } + + + virtual uint32_t readSetEnd() { + uint32_t rv = source_->readSetEnd(); + sink_->writeSetEnd(); + return rv; + } + + virtual uint32_t readBool(bool& value) { + uint32_t rv = source_->readBool(value); + sink_->writeBool(value); + return rv; + } + + virtual uint32_t readByte(int8_t& byte) { + uint32_t rv = source_->readByte(byte); + sink_->writeByte(byte); + return rv; + } + + virtual uint32_t readI16(int16_t& i16) { + uint32_t rv = source_->readI16(i16); + sink_->writeI16(i16); + return rv; + } + + virtual uint32_t readI32(int32_t& i32) { + uint32_t rv = source_->readI32(i32); + sink_->writeI32(i32); + return rv; + } + + virtual uint32_t readI64(int64_t& i64) { + uint32_t rv = source_->readI64(i64); + sink_->writeI64(i64); + return rv; + } + + virtual uint32_t readDouble(double& dub) { + uint32_t rv = source_->readDouble(dub); + sink_->writeDouble(dub); + return rv; + } + + virtual uint32_t readString(std::string& str) { + uint32_t rv = source_->readString(str); + sink_->writeString(str); + return rv; + } + + virtual uint32_t readBinary(std::string& str) { + uint32_t rv = source_->readBinary(str); + sink_->writeBinary(str); + return rv; + } + + private: + boost::shared_ptr source_; + boost::shared_ptr sink_; +}; + +}}} // apache::thrift::protocol + +#endif // #define _THRIFT_PROTOCOL_TPROTOCOLTAP_H_ 1 diff --git a/lib/cpp/src/server/TNonblockingServer.cpp b/lib/cpp/src/server/TNonblockingServer.cpp new file mode 100644 index 00000000..45f635cb --- /dev/null +++ b/lib/cpp/src/server/TNonblockingServer.cpp @@ -0,0 +1,750 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "TNonblockingServer.h" +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace server { + +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace apache::thrift::concurrency; +using namespace std; + +class TConnection::Task: public Runnable { + public: + Task(boost::shared_ptr processor, + boost::shared_ptr input, + boost::shared_ptr output, + int taskHandle) : + processor_(processor), + input_(input), + output_(output), + taskHandle_(taskHandle) {} + + void run() { + try { + while (processor_->process(input_, output_)) { + if (!input_->getTransport()->peek()) { + break; + } + } + } catch (TTransportException& ttx) { + cerr << "TNonblockingServer client died: " << ttx.what() << endl; + } catch (TException& x) { + cerr << "TNonblockingServer exception: " << x.what() << endl; + } catch (...) { + cerr << "TNonblockingServer uncaught exception." << endl; + } + + // Signal completion back to the libevent thread via a socketpair + int8_t b = 0; + if (-1 == send(taskHandle_, &b, sizeof(int8_t), 0)) { + GlobalOutput.perror("TNonblockingServer::Task: send ", errno); + } + if (-1 == ::close(taskHandle_)) { + GlobalOutput.perror("TNonblockingServer::Task: close, possible resource leak ", errno); + } + } + + private: + boost::shared_ptr processor_; + boost::shared_ptr input_; + boost::shared_ptr output_; + int taskHandle_; +}; + +void TConnection::init(int socket, short eventFlags, TNonblockingServer* s) { + socket_ = socket; + server_ = s; + appState_ = APP_INIT; + eventFlags_ = 0; + + readBufferPos_ = 0; + readWant_ = 0; + + writeBuffer_ = NULL; + writeBufferSize_ = 0; + writeBufferPos_ = 0; + + socketState_ = SOCKET_RECV; + appState_ = APP_INIT; + + taskHandle_ = -1; + + // Set flags, which also registers the event + setFlags(eventFlags); + + // get input/transports + factoryInputTransport_ = s->getInputTransportFactory()->getTransport(inputTransport_); + factoryOutputTransport_ = s->getOutputTransportFactory()->getTransport(outputTransport_); + + // Create protocol + inputProtocol_ = s->getInputProtocolFactory()->getProtocol(factoryInputTransport_); + outputProtocol_ = s->getOutputProtocolFactory()->getProtocol(factoryOutputTransport_); +} + +void TConnection::workSocket() { + int flags=0, got=0, left=0, sent=0; + uint32_t fetch = 0; + + switch (socketState_) { + case SOCKET_RECV: + // It is an error to be in this state if we already have all the data + assert(readBufferPos_ < readWant_); + + // Double the buffer size until it is big enough + if (readWant_ > readBufferSize_) { + while (readWant_ > readBufferSize_) { + readBufferSize_ *= 2; + } + readBuffer_ = (uint8_t*)std::realloc(readBuffer_, readBufferSize_); + if (readBuffer_ == NULL) { + GlobalOutput("TConnection::workSocket() realloc"); + close(); + return; + } + } + + // Read from the socket + fetch = readWant_ - readBufferPos_; + got = recv(socket_, readBuffer_ + readBufferPos_, fetch, 0); + + if (got > 0) { + // Move along in the buffer + readBufferPos_ += got; + + // Check that we did not overdo it + assert(readBufferPos_ <= readWant_); + + // We are done reading, move onto the next state + if (readBufferPos_ == readWant_) { + transition(); + } + return; + } else if (got == -1) { + // Blocking errors are okay, just move on + if (errno == EAGAIN || errno == EWOULDBLOCK) { + return; + } + + if (errno != ECONNRESET) { + GlobalOutput.perror("TConnection::workSocket() recv -1 ", errno); + } + } + + // Whenever we get down here it means a remote disconnect + close(); + + return; + + case SOCKET_SEND: + // Should never have position past size + assert(writeBufferPos_ <= writeBufferSize_); + + // If there is no data to send, then let us move on + if (writeBufferPos_ == writeBufferSize_) { + GlobalOutput("WARNING: Send state with no data to send\n"); + transition(); + return; + } + + flags = 0; + #ifdef MSG_NOSIGNAL + // Note the use of MSG_NOSIGNAL to suppress SIGPIPE errors, instead we + // check for the EPIPE return condition and close the socket in that case + flags |= MSG_NOSIGNAL; + #endif // ifdef MSG_NOSIGNAL + + left = writeBufferSize_ - writeBufferPos_; + sent = send(socket_, writeBuffer_ + writeBufferPos_, left, flags); + + if (sent <= 0) { + // Blocking errors are okay, just move on + if (errno == EAGAIN || errno == EWOULDBLOCK) { + return; + } + if (errno != EPIPE) { + GlobalOutput.perror("TConnection::workSocket() send -1 ", errno); + } + close(); + return; + } + + writeBufferPos_ += sent; + + // Did we overdo it? + assert(writeBufferPos_ <= writeBufferSize_); + + // We are done! + if (writeBufferPos_ == writeBufferSize_) { + transition(); + } + + return; + + default: + GlobalOutput.printf("Shit Got Ill. Socket State %d", socketState_); + assert(0); + } +} + +/** + * This is called when the application transitions from one state into + * another. This means that it has finished writing the data that it needed + * to, or finished receiving the data that it needed to. + */ +void TConnection::transition() { + + int sz = 0; + + // Switch upon the state that we are currently in and move to a new state + switch (appState_) { + + case APP_READ_REQUEST: + // We are done reading the request, package the read buffer into transport + // and get back some data from the dispatch function + // If we've used these transport buffers enough times, reset them to avoid bloating + + inputTransport_->resetBuffer(readBuffer_, readBufferPos_); + ++numReadsSinceReset_; + if (numWritesSinceReset_ < 512) { + outputTransport_->resetBuffer(); + } else { + // reset the capacity of the output transport if we used it enough times that it might be bloated + try { + outputTransport_->resetBuffer(true); + numWritesSinceReset_ = 0; + } catch (TTransportException &ttx) { + GlobalOutput.printf("TTransportException: TMemoryBuffer::resetBuffer() %s", ttx.what()); + close(); + return; + } + } + + // Prepend four bytes of blank space to the buffer so we can + // write the frame size there later. + outputTransport_->getWritePtr(4); + outputTransport_->wroteBytes(4); + + if (server_->isThreadPoolProcessing()) { + // We are setting up a Task to do this work and we will wait on it + int sv[2]; + if (-1 == socketpair(AF_LOCAL, SOCK_STREAM, 0, sv)) { + GlobalOutput.perror("TConnection::socketpair() failed ", errno); + // Now we will fall through to the APP_WAIT_TASK block with no response + } else { + // Create task and dispatch to the thread manager + boost::shared_ptr task = + boost::shared_ptr(new Task(server_->getProcessor(), + inputProtocol_, + outputProtocol_, + sv[1])); + // The application is now waiting on the task to finish + appState_ = APP_WAIT_TASK; + + // Create an event to be notified when the task finishes + event_set(&taskEvent_, + taskHandle_ = sv[0], + EV_READ, + TConnection::taskHandler, + this); + + // Attach to the base + event_base_set(server_->getEventBase(), &taskEvent_); + + // Add the event and start up the server + if (-1 == event_add(&taskEvent_, 0)) { + GlobalOutput("TNonblockingServer::serve(): coult not event_add"); + return; + } + try { + server_->addTask(task); + } catch (IllegalStateException & ise) { + // The ThreadManager is not ready to handle any more tasks (it's probably shutting down). + GlobalOutput.printf("IllegalStateException: Server::process() %s", ise.what()); + close(); + } + + // Set this connection idle so that libevent doesn't process more + // data on it while we're still waiting for the threadmanager to + // finish this task + setIdle(); + return; + } + } else { + try { + // Invoke the processor + server_->getProcessor()->process(inputProtocol_, outputProtocol_); + } catch (TTransportException &ttx) { + GlobalOutput.printf("TTransportException: Server::process() %s", ttx.what()); + close(); + return; + } catch (TException &x) { + GlobalOutput.printf("TException: Server::process() %s", x.what()); + close(); + return; + } catch (...) { + GlobalOutput.printf("Server::process() unknown exception"); + close(); + return; + } + } + + // Intentionally fall through here, the call to process has written into + // the writeBuffer_ + + case APP_WAIT_TASK: + // We have now finished processing a task and the result has been written + // into the outputTransport_, so we grab its contents and place them into + // the writeBuffer_ for actual writing by the libevent thread + + // Get the result of the operation + outputTransport_->getBuffer(&writeBuffer_, &writeBufferSize_); + + // If the function call generated return data, then move into the send + // state and get going + // 4 bytes were reserved for frame size + if (writeBufferSize_ > 4) { + + // Move into write state + writeBufferPos_ = 0; + socketState_ = SOCKET_SEND; + + // Put the frame size into the write buffer + int32_t frameSize = (int32_t)htonl(writeBufferSize_ - 4); + memcpy(writeBuffer_, &frameSize, 4); + + // Socket into write mode + appState_ = APP_SEND_RESULT; + setWrite(); + + // Try to work the socket immediately + // workSocket(); + + return; + } + + // In this case, the request was oneway and we should fall through + // right back into the read frame header state + goto LABEL_APP_INIT; + + case APP_SEND_RESULT: + + ++numWritesSinceReset_; + + // N.B.: We also intentionally fall through here into the INIT state! + + LABEL_APP_INIT: + case APP_INIT: + + // reset the input buffer if we used it enough times that it might be bloated + if (numReadsSinceReset_ > 512) + { + void * new_buffer = std::realloc(readBuffer_, 1024); + if (new_buffer == NULL) { + GlobalOutput("TConnection::transition() realloc"); + close(); + return; + } + readBuffer_ = (uint8_t*) new_buffer; + readBufferSize_ = 1024; + numReadsSinceReset_ = 0; + } + + // Clear write buffer variables + writeBuffer_ = NULL; + writeBufferPos_ = 0; + writeBufferSize_ = 0; + + // Set up read buffer for getting 4 bytes + readBufferPos_ = 0; + readWant_ = 4; + + // Into read4 state we go + socketState_ = SOCKET_RECV; + appState_ = APP_READ_FRAME_SIZE; + + // Register read event + setRead(); + + // Try to work the socket right away + // workSocket(); + + return; + + case APP_READ_FRAME_SIZE: + // We just read the request length, deserialize it + sz = *(int32_t*)readBuffer_; + sz = (int32_t)ntohl(sz); + + if (sz <= 0) { + GlobalOutput.printf("TConnection:transition() Negative frame size %d, remote side not using TFramedTransport?", sz); + close(); + return; + } + + // Reset the read buffer + readWant_ = (uint32_t)sz; + readBufferPos_= 0; + + // Move into read request state + appState_ = APP_READ_REQUEST; + + // Work the socket right away + // workSocket(); + + return; + + default: + GlobalOutput.printf("Totally Fucked. Application State %d", appState_); + assert(0); + } +} + +void TConnection::setFlags(short eventFlags) { + // Catch the do nothing case + if (eventFlags_ == eventFlags) { + return; + } + + // Delete a previously existing event + if (eventFlags_ != 0) { + if (event_del(&event_) == -1) { + GlobalOutput("TConnection::setFlags event_del"); + return; + } + } + + // Update in memory structure + eventFlags_ = eventFlags; + + // Do not call event_set if there are no flags + if (!eventFlags_) { + return; + } + + /** + * event_set: + * + * Prepares the event structure &event to be used in future calls to + * event_add() and event_del(). The event will be prepared to call the + * eventHandler using the 'sock' file descriptor to monitor events. + * + * The events can be either EV_READ, EV_WRITE, or both, indicating + * that an application can read or write from the file respectively without + * blocking. + * + * The eventHandler will be called with the file descriptor that triggered + * the event and the type of event which will be one of: EV_TIMEOUT, + * EV_SIGNAL, EV_READ, EV_WRITE. + * + * The additional flag EV_PERSIST makes an event_add() persistent until + * event_del() has been called. + * + * Once initialized, the &event struct can be used repeatedly with + * event_add() and event_del() and does not need to be reinitialized unless + * the eventHandler and/or the argument to it are to be changed. However, + * when an ev structure has been added to libevent using event_add() the + * structure must persist until the event occurs (assuming EV_PERSIST + * is not set) or is removed using event_del(). You may not reuse the same + * ev structure for multiple monitored descriptors; each descriptor needs + * its own ev. + */ + event_set(&event_, socket_, eventFlags_, TConnection::eventHandler, this); + event_base_set(server_->getEventBase(), &event_); + + // Add the event + if (event_add(&event_, 0) == -1) { + GlobalOutput("TConnection::setFlags(): could not event_add"); + } +} + +/** + * Closes a connection + */ +void TConnection::close() { + // Delete the registered libevent + if (event_del(&event_) == -1) { + GlobalOutput("TConnection::close() event_del"); + } + + // Close the socket + if (socket_ > 0) { + ::close(socket_); + } + socket_ = 0; + + // close any factory produced transports + factoryInputTransport_->close(); + factoryOutputTransport_->close(); + + // Give this object back to the server that owns it + server_->returnConnection(this); +} + +void TConnection::checkIdleBufferMemLimit(uint32_t limit) { + if (readBufferSize_ > limit) { + readBufferSize_ = limit; + readBuffer_ = (uint8_t*)std::realloc(readBuffer_, readBufferSize_); + if (readBuffer_ == NULL) { + GlobalOutput("TConnection::checkIdleBufferMemLimit() realloc"); + close(); + } + } +} + +/** + * Creates a new connection either by reusing an object off the stack or + * by allocating a new one entirely + */ +TConnection* TNonblockingServer::createConnection(int socket, short flags) { + // Check the stack + if (connectionStack_.empty()) { + return new TConnection(socket, flags, this); + } else { + TConnection* result = connectionStack_.top(); + connectionStack_.pop(); + result->init(socket, flags, this); + return result; + } +} + +/** + * Returns a connection to the stack + */ +void TNonblockingServer::returnConnection(TConnection* connection) { + if (connectionStackLimit_ && + (connectionStack_.size() >= connectionStackLimit_)) { + delete connection; + } else { + connection->checkIdleBufferMemLimit(idleBufferMemLimit_); + connectionStack_.push(connection); + } +} + +/** + * Server socket had something happen. We accept all waiting client + * connections on fd and assign TConnection objects to handle those requests. + */ +void TNonblockingServer::handleEvent(int fd, short which) { + // Make sure that libevent didn't fuck up the socket handles + assert(fd == serverSocket_); + + // Server socket accepted a new connection + socklen_t addrLen; + struct sockaddr addr; + addrLen = sizeof(addr); + + // Going to accept a new client socket + int clientSocket; + + // Accept as many new clients as possible, even though libevent signaled only + // one, this helps us to avoid having to go back into the libevent engine so + // many times + while ((clientSocket = accept(fd, &addr, &addrLen)) != -1) { + + // Explicitly set this socket to NONBLOCK mode + int flags; + if ((flags = fcntl(clientSocket, F_GETFL, 0)) < 0 || + fcntl(clientSocket, F_SETFL, flags | O_NONBLOCK) < 0) { + GlobalOutput.perror("thriftServerEventHandler: set O_NONBLOCK (fcntl) ", errno); + close(clientSocket); + return; + } + + // Create a new TConnection for this client socket. + TConnection* clientConnection = + createConnection(clientSocket, EV_READ | EV_PERSIST); + + // Fail fast if we could not create a TConnection object + if (clientConnection == NULL) { + GlobalOutput.printf("thriftServerEventHandler: failed TConnection factory"); + close(clientSocket); + return; + } + + // Put this client connection into the proper state + clientConnection->transition(); + } + + // Done looping accept, now we have to make sure the error is due to + // blocking. Any other error is a problem + if (errno != EAGAIN && errno != EWOULDBLOCK) { + GlobalOutput.perror("thriftServerEventHandler: accept() ", errno); + } +} + +/** + * Creates a socket to listen on and binds it to the local port. + */ +void TNonblockingServer::listenSocket() { + int s; + struct addrinfo hints, *res, *res0; + int error; + + char port[sizeof("65536") + 1]; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG; + sprintf(port, "%d", port_); + + // Wildcard address + error = getaddrinfo(NULL, port, &hints, &res0); + if (error) { + string errStr = "TNonblockingServer::serve() getaddrinfo " + string(gai_strerror(error)); + GlobalOutput(errStr.c_str()); + return; + } + + // Pick the ipv6 address first since ipv4 addresses can be mapped + // into ipv6 space. + for (res = res0; res; res = res->ai_next) { + if (res->ai_family == AF_INET6 || res->ai_next == NULL) + break; + } + + // Create the server socket + s = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if (s == -1) { + freeaddrinfo(res0); + throw TException("TNonblockingServer::serve() socket() -1"); + } + + #ifdef IPV6_V6ONLY + int zero = 0; + if (-1 == setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &zero, sizeof(zero))) { + GlobalOutput("TServerSocket::listen() IPV6_V6ONLY"); + } + #endif // #ifdef IPV6_V6ONLY + + + int one = 1; + + // Set reuseaddr to avoid 2MSL delay on server restart + setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); + + if (bind(s, res->ai_addr, res->ai_addrlen) == -1) { + close(s); + freeaddrinfo(res0); + throw TException("TNonblockingServer::serve() bind"); + } + + // Done with the addr info + freeaddrinfo(res0); + + // Set up this file descriptor for listening + listenSocket(s); +} + +/** + * Takes a socket created by listenSocket() and sets various options on it + * to prepare for use in the server. + */ +void TNonblockingServer::listenSocket(int s) { + // Set socket to nonblocking mode + int flags; + if ((flags = fcntl(s, F_GETFL, 0)) < 0 || + fcntl(s, F_SETFL, flags | O_NONBLOCK) < 0) { + close(s); + throw TException("TNonblockingServer::serve() O_NONBLOCK"); + } + + int one = 1; + struct linger ling = {0, 0}; + + // Keepalive to ensure full result flushing + setsockopt(s, SOL_SOCKET, SO_KEEPALIVE, &one, sizeof(one)); + + // Turn linger off to avoid hung sockets + setsockopt(s, SOL_SOCKET, SO_LINGER, &ling, sizeof(ling)); + + // Set TCP nodelay if available, MAC OS X Hack + // See http://lists.danga.com/pipermail/memcached/2005-March/001240.html + #ifndef TCP_NOPUSH + setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); + #endif + + if (listen(s, LISTEN_BACKLOG) == -1) { + close(s); + throw TException("TNonblockingServer::serve() listen"); + } + + // Cool, this socket is good to go, set it as the serverSocket_ + serverSocket_ = s; +} + +/** + * Register the core libevent events onto the proper base. + */ +void TNonblockingServer::registerEvents(event_base* base) { + assert(serverSocket_ != -1); + assert(!eventBase_); + eventBase_ = base; + + // Print some libevent stats + GlobalOutput.printf("libevent %s method %s", + event_get_version(), + event_get_method()); + + // Register the server event + event_set(&serverEvent_, + serverSocket_, + EV_READ | EV_PERSIST, + TNonblockingServer::eventHandler, + this); + event_base_set(eventBase_, &serverEvent_); + + // Add the event and start up the server + if (-1 == event_add(&serverEvent_, 0)) { + throw TException("TNonblockingServer::serve(): coult not event_add"); + } +} + +/** + * Main workhorse function, starts up the server listening on a port and + * loops over the libevent handler. + */ +void TNonblockingServer::serve() { + // Init socket + listenSocket(); + + // Initialize libevent core + registerEvents(static_cast(event_init())); + + // Run the preServe event + if (eventHandler_ != NULL) { + eventHandler_->preServe(); + } + + // Run libevent engine, never returns, invokes calls to eventHandler + event_base_loop(eventBase_, 0); +} + +}}} // apache::thrift::server diff --git a/lib/cpp/src/server/TNonblockingServer.h b/lib/cpp/src/server/TNonblockingServer.h new file mode 100644 index 00000000..1684b64a --- /dev/null +++ b/lib/cpp/src/server/TNonblockingServer.h @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_SERVER_TNONBLOCKINGSERVER_H_ +#define _THRIFT_SERVER_TNONBLOCKINGSERVER_H_ 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace server { + +using apache::thrift::transport::TMemoryBuffer; +using apache::thrift::protocol::TProtocol; +using apache::thrift::concurrency::Runnable; +using apache::thrift::concurrency::ThreadManager; + +// Forward declaration of class +class TConnection; + +/** + * This is a non-blocking server in C++ for high performance that operates a + * single IO thread. It assumes that all incoming requests are framed with a + * 4 byte length indicator and writes out responses using the same framing. + * + * It does not use the TServerTransport framework, but rather has socket + * operations hardcoded for use with select. + * + */ +class TNonblockingServer : public TServer { + private: + + // Listen backlog + static const int LISTEN_BACKLOG = 1024; + + // Default limit on size of idle connection pool + static const size_t CONNECTION_STACK_LIMIT = 1024; + + // Maximum size of buffer allocated to idle connection + static const uint32_t IDLE_BUFFER_MEM_LIMIT = 8192; + + // Server socket file descriptor + int serverSocket_; + + // Port server runs on + int port_; + + // For processing via thread pool, may be NULL + boost::shared_ptr threadManager_; + + // Is thread pool processing? + bool threadPoolProcessing_; + + // The event base for libevent + event_base* eventBase_; + + // Event struct, for use with eventBase_ + struct event serverEvent_; + + // Number of TConnection object we've created + size_t numTConnections_; + + // Limit for how many TConnection objects to cache + size_t connectionStackLimit_; + + /** + * Max read buffer size for an idle connection. When we place an idle + * TConnection into connectionStack_, we insure that its read buffer is + * reduced to this size to insure that idle connections don't hog memory. + */ + uint32_t idleBufferMemLimit_; + + /** + * This is a stack of all the objects that have been created but that + * are NOT currently in use. When we close a connection, we place it on this + * stack so that the object can be reused later, rather than freeing the + * memory and reallocating a new object later. + */ + std::stack connectionStack_; + + void handleEvent(int fd, short which); + + public: + TNonblockingServer(boost::shared_ptr processor, + int port) : + TServer(processor), + serverSocket_(-1), + port_(port), + threadPoolProcessing_(false), + eventBase_(NULL), + numTConnections_(0), + connectionStackLimit_(CONNECTION_STACK_LIMIT), + idleBufferMemLimit_(IDLE_BUFFER_MEM_LIMIT) {} + + TNonblockingServer(boost::shared_ptr processor, + boost::shared_ptr protocolFactory, + int port, + boost::shared_ptr threadManager = boost::shared_ptr()) : + TServer(processor), + serverSocket_(-1), + port_(port), + threadManager_(threadManager), + eventBase_(NULL), + numTConnections_(0), + connectionStackLimit_(CONNECTION_STACK_LIMIT), + idleBufferMemLimit_(IDLE_BUFFER_MEM_LIMIT) { + setInputTransportFactory(boost::shared_ptr(new TTransportFactory())); + setOutputTransportFactory(boost::shared_ptr(new TTransportFactory())); + setInputProtocolFactory(protocolFactory); + setOutputProtocolFactory(protocolFactory); + setThreadManager(threadManager); + } + + TNonblockingServer(boost::shared_ptr processor, + boost::shared_ptr inputTransportFactory, + boost::shared_ptr outputTransportFactory, + boost::shared_ptr inputProtocolFactory, + boost::shared_ptr outputProtocolFactory, + int port, + boost::shared_ptr threadManager = boost::shared_ptr()) : + TServer(processor), + serverSocket_(0), + port_(port), + threadManager_(threadManager), + eventBase_(NULL), + numTConnections_(0), + connectionStackLimit_(CONNECTION_STACK_LIMIT), + idleBufferMemLimit_(IDLE_BUFFER_MEM_LIMIT) { + setInputTransportFactory(inputTransportFactory); + setOutputTransportFactory(outputTransportFactory); + setInputProtocolFactory(inputProtocolFactory); + setOutputProtocolFactory(outputProtocolFactory); + setThreadManager(threadManager); + } + + ~TNonblockingServer() {} + + void setThreadManager(boost::shared_ptr threadManager) { + threadManager_ = threadManager; + threadPoolProcessing_ = (threadManager != NULL); + } + + boost::shared_ptr getThreadManager() { + return threadManager_; + } + + /** + * Get the maximum number of unused TConnection we will hold in reserve. + * + * @return the current limit on TConnection pool size. + */ + size_t getConnectionStackLimit() const { + return connectionStackLimit_; + } + + /** + * Set the maximum number of unused TConnection we will hold in reserve. + * + * @param sz the new limit for TConnection pool size. + */ + void setConnectionStackLimit(size_t sz) { + connectionStackLimit_ = sz; + } + + bool isThreadPoolProcessing() const { + return threadPoolProcessing_; + } + + void addTask(boost::shared_ptr task) { + threadManager_->add(task); + } + + event_base* getEventBase() const { + return eventBase_; + } + + void incrementNumConnections() { + ++numTConnections_; + } + + void decrementNumConnections() { + --numTConnections_; + } + + size_t getNumConnections() { + return numTConnections_; + } + + size_t getNumIdleConnections() { + return connectionStack_.size(); + } + + /** + * Get the maximum limit of memory allocated to idle TConnection objects. + * + * @return # bytes beyond which we will shrink buffers when idle. + */ + size_t getIdleBufferMemLimit() const { + return idleBufferMemLimit_; + } + + /** + * Set the maximum limit of memory allocated to idle TConnection objects. + * If a TConnection object goes idle with more than this much memory + * allocated to its buffer, we shrink it to this value. + * + * @param limit of bytes beyond which we will shrink buffers when idle. + */ + void setIdleBufferMemLimit(size_t limit) { + idleBufferMemLimit_ = limit; + } + + TConnection* createConnection(int socket, short flags); + + void returnConnection(TConnection* connection); + + static void eventHandler(int fd, short which, void* v) { + ((TNonblockingServer*)v)->handleEvent(fd, which); + } + + void listenSocket(); + + void listenSocket(int fd); + + void registerEvents(event_base* base); + + void serve(); +}; + +/** + * Two states for sockets, recv and send mode + */ +enum TSocketState { + SOCKET_RECV, + SOCKET_SEND +}; + +/** + * Four states for the nonblocking servr: + * 1) initialize + * 2) read 4 byte frame size + * 3) read frame of data + * 4) send back data (if any) + */ +enum TAppState { + APP_INIT, + APP_READ_FRAME_SIZE, + APP_READ_REQUEST, + APP_WAIT_TASK, + APP_SEND_RESULT +}; + +/** + * Represents a connection that is handled via libevent. This connection + * essentially encapsulates a socket that has some associated libevent state. + */ +class TConnection { + private: + + class Task; + + // Server handle + TNonblockingServer* server_; + + // Socket handle + int socket_; + + // Libevent object + struct event event_; + + // Libevent flags + short eventFlags_; + + // Socket mode + TSocketState socketState_; + + // Application state + TAppState appState_; + + // How much data needed to read + uint32_t readWant_; + + // Where in the read buffer are we + uint32_t readBufferPos_; + + // Read buffer + uint8_t* readBuffer_; + + // Read buffer size + uint32_t readBufferSize_; + + // Write buffer + uint8_t* writeBuffer_; + + // Write buffer size + uint32_t writeBufferSize_; + + // How far through writing are we? + uint32_t writeBufferPos_; + + // How many times have we read since our last buffer reset? + uint32_t numReadsSinceReset_; + + // How many times have we written since our last buffer reset? + uint32_t numWritesSinceReset_; + + // Task handle + int taskHandle_; + + // Task event + struct event taskEvent_; + + // Transport to read from + boost::shared_ptr inputTransport_; + + // Transport that processor writes to + boost::shared_ptr outputTransport_; + + // extra transport generated by transport factory (e.g. BufferedRouterTransport) + boost::shared_ptr factoryInputTransport_; + boost::shared_ptr factoryOutputTransport_; + + // Protocol decoder + boost::shared_ptr inputProtocol_; + + // Protocol encoder + boost::shared_ptr outputProtocol_; + + // Go into read mode + void setRead() { + setFlags(EV_READ | EV_PERSIST); + } + + // Go into write mode + void setWrite() { + setFlags(EV_WRITE | EV_PERSIST); + } + + // Set socket idle + void setIdle() { + setFlags(0); + } + + // Set event flags + void setFlags(short eventFlags); + + // Libevent handlers + void workSocket(); + + // Close this client and reset + void close(); + + public: + + // Constructor + TConnection(int socket, short eventFlags, TNonblockingServer *s) { + readBuffer_ = (uint8_t*)std::malloc(1024); + if (readBuffer_ == NULL) { + throw new apache::thrift::TException("Out of memory."); + } + readBufferSize_ = 1024; + + numReadsSinceReset_ = 0; + numWritesSinceReset_ = 0; + + // Allocate input and output tranpsorts + // these only need to be allocated once per TConnection (they don't need to be + // reallocated on init() call) + inputTransport_ = boost::shared_ptr(new TMemoryBuffer(readBuffer_, readBufferSize_)); + outputTransport_ = boost::shared_ptr(new TMemoryBuffer()); + + init(socket, eventFlags, s); + server_->incrementNumConnections(); + } + + ~TConnection() { + server_->decrementNumConnections(); + } + + /** + * Check read buffer against a given limit and shrink it if exceeded. + * + * @param limit we limit buffer size to. + */ + void checkIdleBufferMemLimit(uint32_t limit); + + // Initialize + void init(int socket, short eventFlags, TNonblockingServer *s); + + // Transition into a new state + void transition(); + + // Handler wrapper + static void eventHandler(int fd, short /* which */, void* v) { + assert(fd == ((TConnection*)v)->socket_); + ((TConnection*)v)->workSocket(); + } + + // Handler wrapper for task block + static void taskHandler(int fd, short /* which */, void* v) { + assert(fd == ((TConnection*)v)->taskHandle_); + if (-1 == ::close(((TConnection*)v)->taskHandle_)) { + GlobalOutput.perror("TConnection::taskHandler close handle failed, resource leak ", errno); + } + ((TConnection*)v)->transition(); + } + +}; + +}}} // apache::thrift::server + +#endif // #ifndef _THRIFT_SERVER_TSIMPLESERVER_H_ diff --git a/lib/cpp/src/server/TServer.cpp b/lib/cpp/src/server/TServer.cpp new file mode 100644 index 00000000..6b692ab0 --- /dev/null +++ b/lib/cpp/src/server/TServer.cpp @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +namespace apache { namespace thrift { namespace server { + +int increase_max_fds(int max_fds=(1<<24)) { + struct rlimit fdmaxrl; + + for(fdmaxrl.rlim_cur = max_fds, fdmaxrl.rlim_max = max_fds; + max_fds && (setrlimit(RLIMIT_NOFILE, &fdmaxrl) < 0); + fdmaxrl.rlim_cur = max_fds, fdmaxrl.rlim_max = max_fds) { + max_fds /= 2; + } + + return fdmaxrl.rlim_cur; +} + +}}} // apache::thrift::server diff --git a/lib/cpp/src/server/TServer.h b/lib/cpp/src/server/TServer.h new file mode 100644 index 00000000..5c4c588d --- /dev/null +++ b/lib/cpp/src/server/TServer.h @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_SERVER_TSERVER_H_ +#define _THRIFT_SERVER_TSERVER_H_ 1 + +#include +#include +#include +#include + +#include + +namespace apache { namespace thrift { namespace server { + +using apache::thrift::TProcessor; +using apache::thrift::protocol::TBinaryProtocolFactory; +using apache::thrift::protocol::TProtocol; +using apache::thrift::protocol::TProtocolFactory; +using apache::thrift::transport::TServerTransport; +using apache::thrift::transport::TTransport; +using apache::thrift::transport::TTransportFactory; + +/** + * Virtual interface class that can handle events from the server core. To + * use this you should subclass it and implement the methods that you care + * about. Your subclass can also store local data that you may care about, + * such as additional "arguments" to these methods (stored in the object + * instance's state). + */ +class TServerEventHandler { + public: + + virtual ~TServerEventHandler() {} + + /** + * Called before the server begins. + */ + virtual void preServe() {} + + /** + * Called when a new client has connected and is about to being processing. + */ + virtual void clientBegin(boost::shared_ptr /* input */, + boost::shared_ptr /* output */) {} + + /** + * Called when a client has finished making requests. + */ + virtual void clientEnd(boost::shared_ptr /* input */, + boost::shared_ptr /* output */) {} + + protected: + + /** + * Prevent direct instantiation. + */ + TServerEventHandler() {} + +}; + +/** + * Thrift server. + * + */ +class TServer : public concurrency::Runnable { + public: + + virtual ~TServer() {} + + virtual void serve() = 0; + + virtual void stop() {} + + // Allows running the server as a Runnable thread + virtual void run() { + serve(); + } + + boost::shared_ptr getProcessor() { + return processor_; + } + + boost::shared_ptr getServerTransport() { + return serverTransport_; + } + + boost::shared_ptr getInputTransportFactory() { + return inputTransportFactory_; + } + + boost::shared_ptr getOutputTransportFactory() { + return outputTransportFactory_; + } + + boost::shared_ptr getInputProtocolFactory() { + return inputProtocolFactory_; + } + + boost::shared_ptr getOutputProtocolFactory() { + return outputProtocolFactory_; + } + + boost::shared_ptr getEventHandler() { + return eventHandler_; + } + +protected: + TServer(boost::shared_ptr processor): + processor_(processor) { + setInputTransportFactory(boost::shared_ptr(new TTransportFactory())); + setOutputTransportFactory(boost::shared_ptr(new TTransportFactory())); + setInputProtocolFactory(boost::shared_ptr(new TBinaryProtocolFactory())); + setOutputProtocolFactory(boost::shared_ptr(new TBinaryProtocolFactory())); + } + + TServer(boost::shared_ptr processor, + boost::shared_ptr serverTransport): + processor_(processor), + serverTransport_(serverTransport) { + setInputTransportFactory(boost::shared_ptr(new TTransportFactory())); + setOutputTransportFactory(boost::shared_ptr(new TTransportFactory())); + setInputProtocolFactory(boost::shared_ptr(new TBinaryProtocolFactory())); + setOutputProtocolFactory(boost::shared_ptr(new TBinaryProtocolFactory())); + } + + TServer(boost::shared_ptr processor, + boost::shared_ptr serverTransport, + boost::shared_ptr transportFactory, + boost::shared_ptr protocolFactory): + processor_(processor), + serverTransport_(serverTransport), + inputTransportFactory_(transportFactory), + outputTransportFactory_(transportFactory), + inputProtocolFactory_(protocolFactory), + outputProtocolFactory_(protocolFactory) {} + + TServer(boost::shared_ptr processor, + boost::shared_ptr serverTransport, + boost::shared_ptr inputTransportFactory, + boost::shared_ptr outputTransportFactory, + boost::shared_ptr inputProtocolFactory, + boost::shared_ptr outputProtocolFactory): + processor_(processor), + serverTransport_(serverTransport), + inputTransportFactory_(inputTransportFactory), + outputTransportFactory_(outputTransportFactory), + inputProtocolFactory_(inputProtocolFactory), + outputProtocolFactory_(outputProtocolFactory) {} + + + // Class variables + boost::shared_ptr processor_; + boost::shared_ptr serverTransport_; + + boost::shared_ptr inputTransportFactory_; + boost::shared_ptr outputTransportFactory_; + + boost::shared_ptr inputProtocolFactory_; + boost::shared_ptr outputProtocolFactory_; + + boost::shared_ptr eventHandler_; + +public: + void setInputTransportFactory(boost::shared_ptr inputTransportFactory) { + inputTransportFactory_ = inputTransportFactory; + } + + void setOutputTransportFactory(boost::shared_ptr outputTransportFactory) { + outputTransportFactory_ = outputTransportFactory; + } + + void setInputProtocolFactory(boost::shared_ptr inputProtocolFactory) { + inputProtocolFactory_ = inputProtocolFactory; + } + + void setOutputProtocolFactory(boost::shared_ptr outputProtocolFactory) { + outputProtocolFactory_ = outputProtocolFactory; + } + + void setServerEventHandler(boost::shared_ptr eventHandler) { + eventHandler_ = eventHandler; + } + +}; + +/** + * Helper function to increase the max file descriptors limit + * for the current process and all of its children. + * By default, tries to increase it to as much as 2^24. + */ + int increase_max_fds(int max_fds=(1<<24)); + + +}}} // apache::thrift::server + +#endif // #ifndef _THRIFT_SERVER_TSERVER_H_ diff --git a/lib/cpp/src/server/TSimpleServer.cpp b/lib/cpp/src/server/TSimpleServer.cpp new file mode 100644 index 00000000..394ce21e --- /dev/null +++ b/lib/cpp/src/server/TSimpleServer.cpp @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "server/TSimpleServer.h" +#include "transport/TTransportException.h" +#include +#include + +namespace apache { namespace thrift { namespace server { + +using namespace std; +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using boost::shared_ptr; + +/** + * A simple single-threaded application server. Perfect for unit tests! + * + */ +void TSimpleServer::serve() { + + shared_ptr client; + shared_ptr inputTransport; + shared_ptr outputTransport; + shared_ptr inputProtocol; + shared_ptr outputProtocol; + + try { + // Start the server listening + serverTransport_->listen(); + } catch (TTransportException& ttx) { + cerr << "TSimpleServer::run() listen(): " << ttx.what() << endl; + return; + } + + // Run the preServe event + if (eventHandler_ != NULL) { + eventHandler_->preServe(); + } + + // Fetch client from server + while (!stop_) { + try { + client = serverTransport_->accept(); + inputTransport = inputTransportFactory_->getTransport(client); + outputTransport = outputTransportFactory_->getTransport(client); + inputProtocol = inputProtocolFactory_->getProtocol(inputTransport); + outputProtocol = outputProtocolFactory_->getProtocol(outputTransport); + if (eventHandler_ != NULL) { + eventHandler_->clientBegin(inputProtocol, outputProtocol); + } + try { + while (processor_->process(inputProtocol, outputProtocol)) { + // Peek ahead, is the remote side closed? + if (!inputTransport->peek()) { + break; + } + } + } catch (TTransportException& ttx) { + cerr << "TSimpleServer client died: " << ttx.what() << endl; + } catch (TException& tx) { + cerr << "TSimpleServer exception: " << tx.what() << endl; + } + if (eventHandler_ != NULL) { + eventHandler_->clientEnd(inputProtocol, outputProtocol); + } + inputTransport->close(); + outputTransport->close(); + client->close(); + } catch (TTransportException& ttx) { + if (inputTransport != NULL) { inputTransport->close(); } + if (outputTransport != NULL) { outputTransport->close(); } + if (client != NULL) { client->close(); } + cerr << "TServerTransport died on accept: " << ttx.what() << endl; + continue; + } catch (TException& tx) { + if (inputTransport != NULL) { inputTransport->close(); } + if (outputTransport != NULL) { outputTransport->close(); } + if (client != NULL) { client->close(); } + cerr << "Some kind of accept exception: " << tx.what() << endl; + continue; + } catch (string s) { + if (inputTransport != NULL) { inputTransport->close(); } + if (outputTransport != NULL) { outputTransport->close(); } + if (client != NULL) { client->close(); } + cerr << "TThreadPoolServer: Unknown exception: " << s << endl; + break; + } + } + + if (stop_) { + try { + serverTransport_->close(); + } catch (TTransportException &ttx) { + cerr << "TServerTransport failed on close: " << ttx.what() << endl; + } + stop_ = false; + } +} + +}}} // apache::thrift::server diff --git a/lib/cpp/src/server/TSimpleServer.h b/lib/cpp/src/server/TSimpleServer.h new file mode 100644 index 00000000..c4fc91c7 --- /dev/null +++ b/lib/cpp/src/server/TSimpleServer.h @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_SERVER_TSIMPLESERVER_H_ +#define _THRIFT_SERVER_TSIMPLESERVER_H_ 1 + +#include "server/TServer.h" +#include "transport/TServerTransport.h" + +namespace apache { namespace thrift { namespace server { + +/** + * This is the most basic simple server. It is single-threaded and runs a + * continuous loop of accepting a single connection, processing requests on + * that connection until it closes, and then repeating. It is a good example + * of how to extend the TServer interface. + * + */ +class TSimpleServer : public TServer { + public: + TSimpleServer(boost::shared_ptr processor, + boost::shared_ptr serverTransport, + boost::shared_ptr transportFactory, + boost::shared_ptr protocolFactory) : + TServer(processor, serverTransport, transportFactory, protocolFactory), + stop_(false) {} + + TSimpleServer(boost::shared_ptr processor, + boost::shared_ptr serverTransport, + boost::shared_ptr inputTransportFactory, + boost::shared_ptr outputTransportFactory, + boost::shared_ptr inputProtocolFactory, + boost::shared_ptr outputProtocolFactory): + TServer(processor, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory), + stop_(false) {} + + ~TSimpleServer() {} + + void serve(); + + void stop() { + stop_ = true; + } + + protected: + bool stop_; + +}; + +}}} // apache::thrift::server + +#endif // #ifndef _THRIFT_SERVER_TSIMPLESERVER_H_ diff --git a/lib/cpp/src/server/TThreadPoolServer.cpp b/lib/cpp/src/server/TThreadPoolServer.cpp new file mode 100644 index 00000000..0894cfa5 --- /dev/null +++ b/lib/cpp/src/server/TThreadPoolServer.cpp @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "server/TThreadPoolServer.h" +#include "transport/TTransportException.h" +#include "concurrency/Thread.h" +#include "concurrency/ThreadManager.h" +#include +#include + +namespace apache { namespace thrift { namespace server { + +using boost::shared_ptr; +using namespace std; +using namespace apache::thrift; +using namespace apache::thrift::concurrency; +using namespace apache::thrift::protocol;; +using namespace apache::thrift::transport; + +class TThreadPoolServer::Task : public Runnable { + +public: + + Task(TThreadPoolServer &server, + shared_ptr processor, + shared_ptr input, + shared_ptr output) : + server_(server), + processor_(processor), + input_(input), + output_(output) { + } + + ~Task() {} + + void run() { + boost::shared_ptr eventHandler = + server_.getEventHandler(); + if (eventHandler != NULL) { + eventHandler->clientBegin(input_, output_); + } + try { + while (processor_->process(input_, output_)) { + if (!input_->getTransport()->peek()) { + break; + } + } + } catch (TTransportException& ttx) { + // This is reasonably expected, client didn't send a full request so just + // ignore him + // string errStr = string("TThreadPoolServer client died: ") + ttx.what(); + // GlobalOutput(errStr.c_str()); + } catch (TException& x) { + string errStr = string("TThreadPoolServer exception: ") + x.what(); + GlobalOutput(errStr.c_str()); + } catch (std::exception &x) { + string errStr = string("TThreadPoolServer, std::exception: ") + x.what(); + GlobalOutput(errStr.c_str()); + } + + if (eventHandler != NULL) { + eventHandler->clientEnd(input_, output_); + } + + try { + input_->getTransport()->close(); + } catch (TTransportException& ttx) { + string errStr = string("TThreadPoolServer input close failed: ") + ttx.what(); + GlobalOutput(errStr.c_str()); + } + try { + output_->getTransport()->close(); + } catch (TTransportException& ttx) { + string errStr = string("TThreadPoolServer output close failed: ") + ttx.what(); + GlobalOutput(errStr.c_str()); + } + + } + + private: + TServer& server_; + shared_ptr processor_; + shared_ptr input_; + shared_ptr output_; + +}; + +TThreadPoolServer::TThreadPoolServer(shared_ptr processor, + shared_ptr serverTransport, + shared_ptr transportFactory, + shared_ptr protocolFactory, + shared_ptr threadManager) : + TServer(processor, serverTransport, transportFactory, protocolFactory), + threadManager_(threadManager), + stop_(false), timeout_(0) {} + +TThreadPoolServer::TThreadPoolServer(shared_ptr processor, + shared_ptr serverTransport, + shared_ptr inputTransportFactory, + shared_ptr outputTransportFactory, + shared_ptr inputProtocolFactory, + shared_ptr outputProtocolFactory, + shared_ptr threadManager) : + TServer(processor, serverTransport, inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory), + threadManager_(threadManager), + stop_(false), timeout_(0) {} + + +TThreadPoolServer::~TThreadPoolServer() {} + +void TThreadPoolServer::serve() { + shared_ptr client; + shared_ptr inputTransport; + shared_ptr outputTransport; + shared_ptr inputProtocol; + shared_ptr outputProtocol; + + try { + // Start the server listening + serverTransport_->listen(); + } catch (TTransportException& ttx) { + string errStr = string("TThreadPoolServer::run() listen(): ") + ttx.what(); + GlobalOutput(errStr.c_str()); + return; + } + + // Run the preServe event + if (eventHandler_ != NULL) { + eventHandler_->preServe(); + } + + while (!stop_) { + try { + client.reset(); + inputTransport.reset(); + outputTransport.reset(); + inputProtocol.reset(); + outputProtocol.reset(); + + // Fetch client from server + client = serverTransport_->accept(); + + // Make IO transports + inputTransport = inputTransportFactory_->getTransport(client); + outputTransport = outputTransportFactory_->getTransport(client); + inputProtocol = inputProtocolFactory_->getProtocol(inputTransport); + outputProtocol = outputProtocolFactory_->getProtocol(outputTransport); + + // Add to threadmanager pool + threadManager_->add(shared_ptr(new TThreadPoolServer::Task(*this, processor_, inputProtocol, outputProtocol)), timeout_); + + } catch (TTransportException& ttx) { + if (inputTransport != NULL) { inputTransport->close(); } + if (outputTransport != NULL) { outputTransport->close(); } + if (client != NULL) { client->close(); } + if (!stop_ || ttx.getType() != TTransportException::INTERRUPTED) { + string errStr = string("TThreadPoolServer: TServerTransport died on accept: ") + ttx.what(); + GlobalOutput(errStr.c_str()); + } + continue; + } catch (TException& tx) { + if (inputTransport != NULL) { inputTransport->close(); } + if (outputTransport != NULL) { outputTransport->close(); } + if (client != NULL) { client->close(); } + string errStr = string("TThreadPoolServer: Caught TException: ") + tx.what(); + GlobalOutput(errStr.c_str()); + continue; + } catch (string s) { + if (inputTransport != NULL) { inputTransport->close(); } + if (outputTransport != NULL) { outputTransport->close(); } + if (client != NULL) { client->close(); } + string errStr = "TThreadPoolServer: Unknown exception: " + s; + GlobalOutput(errStr.c_str()); + break; + } + } + + // If stopped manually, join the existing threads + if (stop_) { + try { + serverTransport_->close(); + threadManager_->join(); + } catch (TException &tx) { + string errStr = string("TThreadPoolServer: Exception shutting down: ") + tx.what(); + GlobalOutput(errStr.c_str()); + } + stop_ = false; + } + +} + +int64_t TThreadPoolServer::getTimeout() const { + return timeout_; +} + +void TThreadPoolServer::setTimeout(int64_t value) { + timeout_ = value; +} + +}}} // apache::thrift::server diff --git a/lib/cpp/src/server/TThreadPoolServer.h b/lib/cpp/src/server/TThreadPoolServer.h new file mode 100644 index 00000000..7b7e9064 --- /dev/null +++ b/lib/cpp/src/server/TThreadPoolServer.h @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_SERVER_TTHREADPOOLSERVER_H_ +#define _THRIFT_SERVER_TTHREADPOOLSERVER_H_ 1 + +#include +#include +#include + +#include + +namespace apache { namespace thrift { namespace server { + +using apache::thrift::concurrency::ThreadManager; +using apache::thrift::protocol::TProtocolFactory; +using apache::thrift::transport::TServerTransport; +using apache::thrift::transport::TTransportFactory; + +class TThreadPoolServer : public TServer { + public: + class Task; + + TThreadPoolServer(boost::shared_ptr processor, + boost::shared_ptr serverTransport, + boost::shared_ptr transportFactory, + boost::shared_ptr protocolFactory, + boost::shared_ptr threadManager); + + TThreadPoolServer(boost::shared_ptr processor, + boost::shared_ptr serverTransport, + boost::shared_ptr inputTransportFactory, + boost::shared_ptr outputTransportFactory, + boost::shared_ptr inputProtocolFactory, + boost::shared_ptr outputProtocolFactory, + boost::shared_ptr threadManager); + + virtual ~TThreadPoolServer(); + + virtual void serve(); + + virtual int64_t getTimeout() const; + + virtual void setTimeout(int64_t value); + + virtual void stop() { + stop_ = true; + serverTransport_->interrupt(); + } + + protected: + + boost::shared_ptr threadManager_; + + volatile bool stop_; + + volatile int64_t timeout_; + +}; + +}}} // apache::thrift::server + +#endif // #ifndef _THRIFT_SERVER_TTHREADPOOLSERVER_H_ diff --git a/lib/cpp/src/server/TThreadedServer.cpp b/lib/cpp/src/server/TThreadedServer.cpp new file mode 100644 index 00000000..cc30f8ff --- /dev/null +++ b/lib/cpp/src/server/TThreadedServer.cpp @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "server/TThreadedServer.h" +#include "transport/TTransportException.h" +#include "concurrency/PosixThreadFactory.h" + +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace server { + +using boost::shared_ptr; +using namespace std; +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace apache::thrift::concurrency; + +class TThreadedServer::Task: public Runnable { + +public: + + Task(TThreadedServer& server, + shared_ptr processor, + shared_ptr input, + shared_ptr output) : + server_(server), + processor_(processor), + input_(input), + output_(output) { + } + + ~Task() {} + + void run() { + boost::shared_ptr eventHandler = + server_.getEventHandler(); + if (eventHandler != NULL) { + eventHandler->clientBegin(input_, output_); + } + try { + while (processor_->process(input_, output_)) { + if (!input_->getTransport()->peek()) { + break; + } + } + } catch (TTransportException& ttx) { + string errStr = string("TThreadedServer client died: ") + ttx.what(); + GlobalOutput(errStr.c_str()); + } catch (TException& x) { + string errStr = string("TThreadedServer exception: ") + x.what(); + GlobalOutput(errStr.c_str()); + } catch (...) { + GlobalOutput("TThreadedServer uncaught exception."); + } + if (eventHandler != NULL) { + eventHandler->clientEnd(input_, output_); + } + + try { + input_->getTransport()->close(); + } catch (TTransportException& ttx) { + string errStr = string("TThreadedServer input close failed: ") + ttx.what(); + GlobalOutput(errStr.c_str()); + } + try { + output_->getTransport()->close(); + } catch (TTransportException& ttx) { + string errStr = string("TThreadedServer output close failed: ") + ttx.what(); + GlobalOutput(errStr.c_str()); + } + + // Remove this task from parent bookkeeping + { + Synchronized s(server_.tasksMonitor_); + server_.tasks_.erase(this); + if (server_.tasks_.empty()) { + server_.tasksMonitor_.notify(); + } + } + + } + + private: + TThreadedServer& server_; + friend class TThreadedServer; + + shared_ptr processor_; + shared_ptr input_; + shared_ptr output_; +}; + + +TThreadedServer::TThreadedServer(shared_ptr processor, + shared_ptr serverTransport, + shared_ptr transportFactory, + shared_ptr protocolFactory): + TServer(processor, serverTransport, transportFactory, protocolFactory), + stop_(false) { + threadFactory_ = shared_ptr(new PosixThreadFactory()); +} + +TThreadedServer::TThreadedServer(boost::shared_ptr processor, + boost::shared_ptr serverTransport, + boost::shared_ptr transportFactory, + boost::shared_ptr protocolFactory, + boost::shared_ptr threadFactory): + TServer(processor, serverTransport, transportFactory, protocolFactory), + threadFactory_(threadFactory), + stop_(false) { +} + +TThreadedServer::~TThreadedServer() {} + +void TThreadedServer::serve() { + + shared_ptr client; + shared_ptr inputTransport; + shared_ptr outputTransport; + shared_ptr inputProtocol; + shared_ptr outputProtocol; + + try { + // Start the server listening + serverTransport_->listen(); + } catch (TTransportException& ttx) { + string errStr = string("TThreadedServer::run() listen(): ") +ttx.what(); + GlobalOutput(errStr.c_str()); + return; + } + + // Run the preServe event + if (eventHandler_ != NULL) { + eventHandler_->preServe(); + } + + while (!stop_) { + try { + client.reset(); + inputTransport.reset(); + outputTransport.reset(); + inputProtocol.reset(); + outputProtocol.reset(); + + // Fetch client from server + client = serverTransport_->accept(); + + // Make IO transports + inputTransport = inputTransportFactory_->getTransport(client); + outputTransport = outputTransportFactory_->getTransport(client); + inputProtocol = inputProtocolFactory_->getProtocol(inputTransport); + outputProtocol = outputProtocolFactory_->getProtocol(outputTransport); + + TThreadedServer::Task* task = new TThreadedServer::Task(*this, + processor_, + inputProtocol, + outputProtocol); + + // Create a task + shared_ptr runnable = + shared_ptr(task); + + // Create a thread for this task + shared_ptr thread = + shared_ptr(threadFactory_->newThread(runnable)); + + // Insert thread into the set of threads + { + Synchronized s(tasksMonitor_); + tasks_.insert(task); + } + + // Start the thread! + thread->start(); + + } catch (TTransportException& ttx) { + if (inputTransport != NULL) { inputTransport->close(); } + if (outputTransport != NULL) { outputTransport->close(); } + if (client != NULL) { client->close(); } + if (!stop_ || ttx.getType() != TTransportException::INTERRUPTED) { + string errStr = string("TThreadedServer: TServerTransport died on accept: ") + ttx.what(); + GlobalOutput(errStr.c_str()); + } + continue; + } catch (TException& tx) { + if (inputTransport != NULL) { inputTransport->close(); } + if (outputTransport != NULL) { outputTransport->close(); } + if (client != NULL) { client->close(); } + string errStr = string("TThreadedServer: Caught TException: ") + tx.what(); + GlobalOutput(errStr.c_str()); + continue; + } catch (string s) { + if (inputTransport != NULL) { inputTransport->close(); } + if (outputTransport != NULL) { outputTransport->close(); } + if (client != NULL) { client->close(); } + string errStr = "TThreadedServer: Unknown exception: " + s; + GlobalOutput(errStr.c_str()); + break; + } + } + + // If stopped manually, make sure to close server transport + if (stop_) { + try { + serverTransport_->close(); + } catch (TException &tx) { + string errStr = string("TThreadedServer: Exception shutting down: ") + tx.what(); + GlobalOutput(errStr.c_str()); + } + try { + Synchronized s(tasksMonitor_); + while (!tasks_.empty()) { + tasksMonitor_.wait(); + } + } catch (TException &tx) { + string errStr = string("TThreadedServer: Exception joining workers: ") + tx.what(); + GlobalOutput(errStr.c_str()); + } + stop_ = false; + } + +} + +}}} // apache::thrift::server diff --git a/lib/cpp/src/server/TThreadedServer.h b/lib/cpp/src/server/TThreadedServer.h new file mode 100644 index 00000000..4d0811aa --- /dev/null +++ b/lib/cpp/src/server/TThreadedServer.h @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_SERVER_TTHREADEDSERVER_H_ +#define _THRIFT_SERVER_TTHREADEDSERVER_H_ 1 + +#include +#include +#include +#include + +#include + +namespace apache { namespace thrift { namespace server { + +using apache::thrift::TProcessor; +using apache::thrift::transport::TServerTransport; +using apache::thrift::transport::TTransportFactory; +using apache::thrift::concurrency::Monitor; +using apache::thrift::concurrency::ThreadFactory; + +class TThreadedServer : public TServer { + + public: + class Task; + + TThreadedServer(boost::shared_ptr processor, + boost::shared_ptr serverTransport, + boost::shared_ptr transportFactory, + boost::shared_ptr protocolFactory); + + TThreadedServer(boost::shared_ptr processor, + boost::shared_ptr serverTransport, + boost::shared_ptr transportFactory, + boost::shared_ptr protocolFactory, + boost::shared_ptr threadFactory); + + virtual ~TThreadedServer(); + + virtual void serve(); + + void stop() { + stop_ = true; + serverTransport_->interrupt(); + } + + protected: + boost::shared_ptr threadFactory_; + volatile bool stop_; + + Monitor tasksMonitor_; + std::set tasks_; + +}; + +}}} // apache::thrift::server + +#endif // #ifndef _THRIFT_SERVER_TTHREADEDSERVER_H_ diff --git a/lib/cpp/src/transport/TBufferTransports.cpp b/lib/cpp/src/transport/TBufferTransports.cpp new file mode 100644 index 00000000..7a7e5e92 --- /dev/null +++ b/lib/cpp/src/transport/TBufferTransports.cpp @@ -0,0 +1,370 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include + +using std::string; + +namespace apache { namespace thrift { namespace transport { + + +uint32_t TBufferedTransport::readSlow(uint8_t* buf, uint32_t len) { + uint32_t want = len; + uint32_t have = rBound_ - rBase_; + + // We should only take the slow path if we can't satisfy the read + // with the data already in the buffer. + assert(have < want); + + // Copy out whatever we have. + if (have > 0) { + memcpy(buf, rBase_, have); + want -= have; + buf += have; + } + // Get more from underlying transport up to buffer size. + // Note that this makes a lot of sense if len < rBufSize_ + // and almost no sense otherwise. TODO(dreiss): Fix that + // case (possibly including some readv hotness). + setReadBuffer(rBuf_.get(), transport_->read(rBuf_.get(), rBufSize_)); + + // Hand over whatever we have. + uint32_t give = std::min(want, static_cast(rBound_ - rBase_)); + memcpy(buf, rBase_, give); + rBase_ += give; + want -= give; + + return (len - want); +} + +void TBufferedTransport::writeSlow(const uint8_t* buf, uint32_t len) { + uint32_t have_bytes = wBase_ - wBuf_.get(); + uint32_t space = wBound_ - wBase_; + // We should only take the slow path if we can't accomodate the write + // with the free space already in the buffer. + assert(wBound_ - wBase_ < static_cast(len)); + + // Now here's the tricky question: should we copy data from buf into our + // internal buffer and write it from there, or should we just write out + // the current internal buffer in one syscall and write out buf in another. + // If our currently buffered data plus buf is at least double our buffer + // size, we will have to do two syscalls no matter what (except in the + // degenerate case when our buffer is empty), so there is no use copying. + // Otherwise, there is sort of a sliding scale. If we have N-1 bytes + // buffered and need to write 2, it would be crazy to do two syscalls. + // On the other hand, if we have 2 bytes buffered and are writing 2N-3, + // we can save a syscall in the short term by loading up our buffer, writing + // it out, and copying the rest of the bytes into our buffer. Of course, + // if we get another 2-byte write, we haven't saved any syscalls at all, + // and have just copied nearly 2N bytes for nothing. Finding a perfect + // policy would require predicting the size of future writes, so we're just + // going to always eschew syscalls if we have less than 2N bytes to write. + + // The case where we have to do two syscalls. + // This case also covers the case where the buffer is empty, + // but it is clearer (I think) to think of it as two separate cases. + if ((have_bytes + len >= 2*wBufSize_) || (have_bytes == 0)) { + // TODO(dreiss): writev + if (have_bytes > 0) { + transport_->write(wBuf_.get(), have_bytes); + } + transport_->write(buf, len); + wBase_ = wBuf_.get(); + return; + } + + // Fill up our internal buffer for a write. + memcpy(wBase_, buf, space); + buf += space; + len -= space; + transport_->write(wBuf_.get(), wBufSize_); + + // Copy the rest into our buffer. + assert(len < wBufSize_); + memcpy(wBuf_.get(), buf, len); + wBase_ = wBuf_.get() + len; + return; +} + +const uint8_t* TBufferedTransport::borrowSlow(uint8_t* buf, uint32_t* len) { + // If the request is bigger than our buffer, we are hosed. + if (*len > rBufSize_) { + return NULL; + } + + // The number of bytes of data we have already. + uint32_t have = rBound_ - rBase_; + // The number of additional bytes we need from the underlying transport. + int32_t need = *len - have; + // The space from the start of the buffer to the end of our data. + uint32_t offset = rBound_ - rBuf_.get(); + assert(need > 0); + + // If we have less than half our buffer space available, shift the data + // we have down to the start. If the borrow is big compared to our buffer, + // this could be kind of a waste, but if the borrow is small, it frees up + // space at the end of our buffer to do a bigger single read from the + // underlying transport. Also, if our needs extend past the end of the + // buffer, we have to do a copy no matter what. + if ((offset > rBufSize_/2) || (offset + need > rBufSize_)) { + memmove(rBuf_.get(), rBase_, have); + setReadBuffer(rBuf_.get(), have); + } + + // First try to fill up the buffer. + uint32_t got = transport_->read(rBound_, rBufSize_ - have); + rBound_ += got; + need -= got; + + // If that fails, readAll until we get what we need. + if (need > 0) { + rBound_ += transport_->readAll(rBound_, need); + } + + *len = rBound_ - rBase_; + return rBase_; +} + +void TBufferedTransport::flush() { + // Write out any data waiting in the write buffer. + uint32_t have_bytes = wBase_ - wBuf_.get(); + if (have_bytes > 0) { + // Note that we reset wBase_ prior to the underlying write + // to ensure we're in a sane state (i.e. internal buffer cleaned) + // if the underlying write throws up an exception + wBase_ = wBuf_.get(); + transport_->write(wBuf_.get(), have_bytes); + } + + // Flush the underlying transport. + transport_->flush(); +} + + +uint32_t TFramedTransport::readSlow(uint8_t* buf, uint32_t len) { + uint32_t want = len; + uint32_t have = rBound_ - rBase_; + + // We should only take the slow path if we can't satisfy the read + // with the data already in the buffer. + assert(have < want); + + // Copy out whatever we have. + if (have > 0) { + memcpy(buf, rBase_, have); + want -= have; + buf += have; + } + + // Read another frame. + readFrame(); + + // TODO(dreiss): Should we warn when reads cross frames? + + // Hand over whatever we have. + uint32_t give = std::min(want, static_cast(rBound_ - rBase_)); + memcpy(buf, rBase_, give); + rBase_ += give; + want -= give; + + return (len - want); +} + +void TFramedTransport::readFrame() { + // TODO(dreiss): Think about using readv here, even though it would + // result in (gasp) read-ahead. + + // Read the size of the next frame. + int32_t sz; + transport_->readAll((uint8_t*)&sz, sizeof(sz)); + sz = ntohl(sz); + + if (sz < 0) { + throw TTransportException("Frame size has negative value"); + } + + // Read the frame payload, and reset markers. + if (sz > static_cast(rBufSize_)) { + rBuf_.reset(new uint8_t[sz]); + rBufSize_ = sz; + } + transport_->readAll(rBuf_.get(), sz); + setReadBuffer(rBuf_.get(), sz); +} + +void TFramedTransport::writeSlow(const uint8_t* buf, uint32_t len) { + // Double buffer size until sufficient. + uint32_t have = wBase_ - wBuf_.get(); + while (wBufSize_ < len + have) { + wBufSize_ *= 2; + } + + // TODO(dreiss): Consider modifying this class to use malloc/free + // so we can use realloc here. + + // Allocate new buffer. + uint8_t* new_buf = new uint8_t[wBufSize_]; + + // Copy the old buffer to the new one. + memcpy(new_buf, wBuf_.get(), have); + + // Now point buf to the new one. + wBuf_.reset(new_buf); + wBase_ = wBuf_.get() + have; + wBound_ = wBuf_.get() + wBufSize_; + + // Copy the data into the new buffer. + memcpy(wBase_, buf, len); + wBase_ += len; +} + +void TFramedTransport::flush() { + int32_t sz_hbo, sz_nbo; + assert(wBufSize_ > sizeof(sz_nbo)); + + // Slip the frame size into the start of the buffer. + sz_hbo = wBase_ - (wBuf_.get() + sizeof(sz_nbo)); + sz_nbo = (int32_t)htonl((uint32_t)(sz_hbo)); + memcpy(wBuf_.get(), (uint8_t*)&sz_nbo, sizeof(sz_nbo)); + + if (sz_hbo > 0) { + // Note that we reset wBase_ (with a pad for the frame size) + // prior to the underlying write to ensure we're in a sane state + // (i.e. internal buffer cleaned) if the underlying write throws + // up an exception + wBase_ = wBuf_.get() + sizeof(sz_nbo); + + // Write size and frame body. + transport_->write(wBuf_.get(), sizeof(sz_nbo)+sz_hbo); + } + + // Flush the underlying transport. + transport_->flush(); +} + +const uint8_t* TFramedTransport::borrowSlow(uint8_t* buf, uint32_t* len) { + // Don't try to be clever with shifting buffers. + // If the fast path failed let the protocol use its slow path. + // Besides, who is going to try to borrow across messages? + return NULL; +} + + +void TMemoryBuffer::computeRead(uint32_t len, uint8_t** out_start, uint32_t* out_give) { + // Correct rBound_ so we can use the fast path in the future. + rBound_ = wBase_; + + // Decide how much to give. + uint32_t give = std::min(len, available_read()); + + *out_start = rBase_; + *out_give = give; + + // Preincrement rBase_ so the caller doesn't have to. + rBase_ += give; +} + +uint32_t TMemoryBuffer::readSlow(uint8_t* buf, uint32_t len) { + uint8_t* start; + uint32_t give; + computeRead(len, &start, &give); + + // Copy into the provided buffer. + memcpy(buf, start, give); + + return give; +} + +uint32_t TMemoryBuffer::readAppendToString(std::string& str, uint32_t len) { + // Don't get some stupid assertion failure. + if (buffer_ == NULL) { + return 0; + } + + uint8_t* start; + uint32_t give; + computeRead(len, &start, &give); + + // Append to the provided string. + str.append((char*)start, give); + + return give; +} + +void TMemoryBuffer::ensureCanWrite(uint32_t len) { + // Check available space + uint32_t avail = available_write(); + if (len <= avail) { + return; + } + + if (!owner_) { + throw TTransportException("Insufficient space in external MemoryBuffer"); + } + + // Grow the buffer as necessary. + while (len > avail) { + bufferSize_ *= 2; + wBound_ = buffer_ + bufferSize_; + avail = available_write(); + } + + // Allocate into a new pointer so we don't bork ours if it fails. + void* new_buffer = std::realloc(buffer_, bufferSize_); + if (new_buffer == NULL) { + throw TTransportException("Out of memory."); + } + + ptrdiff_t offset = (uint8_t*)new_buffer - buffer_; + buffer_ += offset; + rBase_ += offset; + rBound_ += offset; + wBase_ += offset; + wBound_ += offset; +} + +void TMemoryBuffer::writeSlow(const uint8_t* buf, uint32_t len) { + ensureCanWrite(len); + + // Copy into the buffer and increment wBase_. + memcpy(wBase_, buf, len); + wBase_ += len; +} + +void TMemoryBuffer::wroteBytes(uint32_t len) { + uint32_t avail = available_write(); + if (len > avail) { + throw TTransportException("Client wrote more bytes than size of buffer."); + } + wBase_ += len; +} + +const uint8_t* TMemoryBuffer::borrowSlow(uint8_t* buf, uint32_t* len) { + rBound_ = wBase_; + if (available_read() >= *len) { + *len = available_read(); + return rBase_; + } + return NULL; +} + +}}} // apache::thrift::transport diff --git a/lib/cpp/src/transport/TBufferTransports.h b/lib/cpp/src/transport/TBufferTransports.h new file mode 100644 index 00000000..1908205f --- /dev/null +++ b/lib/cpp/src/transport/TBufferTransports.h @@ -0,0 +1,667 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_ +#define _THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_ 1 + +#include +#include "boost/scoped_array.hpp" + +#include + +#ifdef __GNUC__ +#define TDB_LIKELY(val) (__builtin_expect((val), 1)) +#define TDB_UNLIKELY(val) (__builtin_expect((val), 0)) +#else +#define TDB_LIKELY(val) (val) +#define TDB_UNLIKELY(val) (val) +#endif + +namespace apache { namespace thrift { namespace transport { + + +/** + * Base class for all transports that use read/write buffers for performance. + * + * TBufferBase is designed to implement the fast-path "memcpy" style + * operations that work in the common case. It does so with small and + * (eventually) nonvirtual, inlinable methods. TBufferBase is an abstract + * class. Subclasses are expected to define the "slow path" operations + * that have to be done when the buffers are full or empty. + * + */ +class TBufferBase : public TTransport { + + public: + + /** + * Fast-path read. + * + * When we have enough data buffered to fulfill the read, we can satisfy it + * with a single memcpy, then adjust our internal pointers. If the buffer + * is empty, we call out to our slow path, implemented by a subclass. + * This method is meant to eventually be nonvirtual and inlinable. + */ + uint32_t read(uint8_t* buf, uint32_t len) { + uint8_t* new_rBase = rBase_ + len; + if (TDB_LIKELY(new_rBase <= rBound_)) { + std::memcpy(buf, rBase_, len); + rBase_ = new_rBase; + return len; + } + return readSlow(buf, len); + } + + /** + * Fast-path write. + * + * When we have enough empty space in our buffer to accomodate the write, we + * can satisfy it with a single memcpy, then adjust our internal pointers. + * If the buffer is full, we call out to our slow path, implemented by a + * subclass. This method is meant to eventually be nonvirtual and + * inlinable. + */ + void write(const uint8_t* buf, uint32_t len) { + uint8_t* new_wBase = wBase_ + len; + if (TDB_LIKELY(new_wBase <= wBound_)) { + std::memcpy(wBase_, buf, len); + wBase_ = new_wBase; + return; + } + writeSlow(buf, len); + } + + /** + * Fast-path borrow. A lot like the fast-path read. + */ + const uint8_t* borrow(uint8_t* buf, uint32_t* len) { + if (TDB_LIKELY(static_cast(*len) <= rBound_ - rBase_)) { + // With strict aliasing, writing to len shouldn't force us to + // refetch rBase_ from memory. TODO(dreiss): Verify this. + *len = rBound_ - rBase_; + return rBase_; + } + return borrowSlow(buf, len); + } + + /** + * Consume doesn't require a slow path. + */ + void consume(uint32_t len) { + if (TDB_LIKELY(static_cast(len) <= rBound_ - rBase_)) { + rBase_ += len; + } else { + throw TTransportException(TTransportException::BAD_ARGS, + "consume did not follow a borrow."); + } + } + + + protected: + + /// Slow path read. + virtual uint32_t readSlow(uint8_t* buf, uint32_t len) = 0; + + /// Slow path write. + virtual void writeSlow(const uint8_t* buf, uint32_t len) = 0; + + /** + * Slow path borrow. + * + * POSTCONDITION: return == NULL || rBound_ - rBase_ >= *len + */ + virtual const uint8_t* borrowSlow(uint8_t* buf, uint32_t* len) = 0; + + /** + * Trivial constructor. + * + * Initialize pointers safely. Constructing is not a very + * performance-sensitive operation, so it is okay to just leave it to + * the concrete class to set up pointers correctly. + */ + TBufferBase() + : rBase_(NULL) + , rBound_(NULL) + , wBase_(NULL) + , wBound_(NULL) + {} + + /// Convenience mutator for setting the read buffer. + void setReadBuffer(uint8_t* buf, uint32_t len) { + rBase_ = buf; + rBound_ = buf+len; + } + + /// Convenience mutator for setting the write buffer. + void setWriteBuffer(uint8_t* buf, uint32_t len) { + wBase_ = buf; + wBound_ = buf+len; + } + + virtual ~TBufferBase() {} + + /// Reads begin here. + uint8_t* rBase_; + /// Reads may extend to just before here. + uint8_t* rBound_; + + /// Writes begin here. + uint8_t* wBase_; + /// Writes may extend to just before here. + uint8_t* wBound_; +}; + + +/** + * Base class for all transport which wraps transport to new one. + */ +class TUnderlyingTransport : public TBufferBase { + public: + static const int DEFAULT_BUFFER_SIZE = 512; + + virtual bool peek() { + return (rBase_ < rBound_) || transport_->peek(); + } + + void open() { + transport_->open(); + } + + bool isOpen() { + return transport_->isOpen(); + } + + void close() { + flush(); + transport_->close(); + } + + boost::shared_ptr getUnderlyingTransport() { + return transport_; + } + + protected: + boost::shared_ptr transport_; + + uint32_t rBufSize_; + uint32_t wBufSize_; + boost::scoped_array rBuf_; + boost::scoped_array wBuf_; + + TUnderlyingTransport(boost::shared_ptr transport, uint32_t sz) + : transport_(transport) + , rBufSize_(sz) + , wBufSize_(sz) + , rBuf_(new uint8_t[rBufSize_]) + , wBuf_(new uint8_t[wBufSize_]) {} + + TUnderlyingTransport(boost::shared_ptr transport) + : transport_(transport) + , rBufSize_(DEFAULT_BUFFER_SIZE) + , wBufSize_(DEFAULT_BUFFER_SIZE) + , rBuf_(new uint8_t[rBufSize_]) + , wBuf_(new uint8_t[wBufSize_]) {} + + TUnderlyingTransport(boost::shared_ptr transport, uint32_t rsz, uint32_t wsz) + : transport_(transport) + , rBufSize_(rsz) + , wBufSize_(wsz) + , rBuf_(new uint8_t[rBufSize_]) + , wBuf_(new uint8_t[wBufSize_]) {} +}; + +/** + * Buffered transport. For reads it will read more data than is requested + * and will serve future data out of a local buffer. For writes, data is + * stored to an in memory buffer before being written out. + * + */ +class TBufferedTransport : public TUnderlyingTransport { + public: + + /// Use default buffer sizes. + TBufferedTransport(boost::shared_ptr transport) + : TUnderlyingTransport(transport) + { + initPointers(); + } + + /// Use specified buffer sizes. + TBufferedTransport(boost::shared_ptr transport, uint32_t sz) + : TUnderlyingTransport(transport, sz) + { + initPointers(); + } + + /// Use specified read and write buffer sizes. + TBufferedTransport(boost::shared_ptr transport, uint32_t rsz, uint32_t wsz) + : TUnderlyingTransport(transport, rsz, wsz) + { + initPointers(); + } + + virtual bool peek() { + /* shigin: see THRIFT-96 discussion */ + if (rBase_ == rBound_) { + setReadBuffer(rBuf_.get(), transport_->read(rBuf_.get(), rBufSize_)); + } + return (rBound_ > rBase_); + } + virtual uint32_t readSlow(uint8_t* buf, uint32_t len); + + virtual void writeSlow(const uint8_t* buf, uint32_t len); + + void flush(); + + + /** + * The following behavior is currently implemented by TBufferedTransport, + * but that may change in a future version: + * 1/ If len is at most rBufSize_, borrow will never return NULL. + * Depending on the underlying transport, it could throw an exception + * or hang forever. + * 2/ Some borrow requests may copy bytes internally. However, + * if len is at most rBufSize_/2, none of the copied bytes + * will ever have to be copied again. For optimial performance, + * stay under this limit. + */ + virtual const uint8_t* borrowSlow(uint8_t* buf, uint32_t* len); + + protected: + void initPointers() { + setReadBuffer(rBuf_.get(), 0); + setWriteBuffer(wBuf_.get(), wBufSize_); + // Write size never changes. + } +}; + + +/** + * Wraps a transport into a buffered one. + * + */ +class TBufferedTransportFactory : public TTransportFactory { + public: + TBufferedTransportFactory() {} + + virtual ~TBufferedTransportFactory() {} + + /** + * Wraps the transport into a buffered one. + */ + virtual boost::shared_ptr getTransport(boost::shared_ptr trans) { + return boost::shared_ptr(new TBufferedTransport(trans)); + } + +}; + + +/** + * Framed transport. All writes go into an in-memory buffer until flush is + * called, at which point the transport writes the length of the entire + * binary chunk followed by the data payload. This allows the receiver on the + * other end to always do fixed-length reads. + * + */ +class TFramedTransport : public TUnderlyingTransport { + public: + + /// Use default buffer sizes. + TFramedTransport(boost::shared_ptr transport) + : TUnderlyingTransport(transport) + { + initPointers(); + } + + TFramedTransport(boost::shared_ptr transport, uint32_t sz) + : TUnderlyingTransport(transport, sz) + { + initPointers(); + } + + virtual uint32_t readSlow(uint8_t* buf, uint32_t len); + + virtual void writeSlow(const uint8_t* buf, uint32_t len); + + virtual void flush(); + + const uint8_t* borrowSlow(uint8_t* buf, uint32_t* len); + + protected: + /** + * Reads a frame of input from the underlying stream. + */ + void readFrame(); + + void initPointers() { + setReadBuffer(NULL, 0); + setWriteBuffer(wBuf_.get(), wBufSize_); + + // Pad the buffer so we can insert the size later. + int32_t pad = 0; + this->write((uint8_t*)&pad, sizeof(pad)); + } +}; + +/** + * Wraps a transport into a framed one. + * + */ +class TFramedTransportFactory : public TTransportFactory { + public: + TFramedTransportFactory() {} + + virtual ~TFramedTransportFactory() {} + + /** + * Wraps the transport into a framed one. + */ + virtual boost::shared_ptr getTransport(boost::shared_ptr trans) { + return boost::shared_ptr(new TFramedTransport(trans)); + } + +}; + + +/** + * A memory buffer is a tranpsort that simply reads from and writes to an + * in memory buffer. Anytime you call write on it, the data is simply placed + * into a buffer, and anytime you call read, data is read from that buffer. + * + * The buffers are allocated using C constructs malloc,realloc, and the size + * doubles as necessary. We've considered using scoped + * + */ +class TMemoryBuffer : public TBufferBase { + private: + + // Common initialization done by all constructors. + void initCommon(uint8_t* buf, uint32_t size, bool owner, uint32_t wPos) { + if (buf == NULL && size != 0) { + assert(owner); + buf = (uint8_t*)std::malloc(size); + if (buf == NULL) { + throw TTransportException("Out of memory"); + } + } + + buffer_ = buf; + bufferSize_ = size; + + rBase_ = buffer_; + rBound_ = buffer_ + wPos; + // TODO(dreiss): Investigate NULL-ing this if !owner. + wBase_ = buffer_ + wPos; + wBound_ = buffer_ + bufferSize_; + + owner_ = owner; + + // rBound_ is really an artifact. In principle, it should always be + // equal to wBase_. We update it in a few places (computeRead, etc.). + } + + public: + static const uint32_t defaultSize = 1024; + + /** + * This enum specifies how a TMemoryBuffer should treat + * memory passed to it via constructors or resetBuffer. + * + * OBSERVE: + * TMemoryBuffer will simply store a pointer to the memory. + * It is the callers responsibility to ensure that the pointer + * remains valid for the lifetime of the TMemoryBuffer, + * and that it is properly cleaned up. + * Note that no data can be written to observed buffers. + * + * COPY: + * TMemoryBuffer will make an internal copy of the buffer. + * The caller has no responsibilities. + * + * TAKE_OWNERSHIP: + * TMemoryBuffer will become the "owner" of the buffer, + * and will be responsible for freeing it. + * The membory must have been allocated with malloc. + */ + enum MemoryPolicy + { OBSERVE = 1 + , COPY = 2 + , TAKE_OWNERSHIP = 3 + }; + + /** + * Construct a TMemoryBuffer with a default-sized buffer, + * owned by the TMemoryBuffer object. + */ + TMemoryBuffer() { + initCommon(NULL, defaultSize, true, 0); + } + + /** + * Construct a TMemoryBuffer with a buffer of a specified size, + * owned by the TMemoryBuffer object. + * + * @param sz The initial size of the buffer. + */ + TMemoryBuffer(uint32_t sz) { + initCommon(NULL, sz, true, 0); + } + + /** + * Construct a TMemoryBuffer with buf as its initial contents. + * + * @param buf The initial contents of the buffer. + * Note that, while buf is a non-const pointer, + * TMemoryBuffer will not write to it if policy == OBSERVE, + * so it is safe to const_cast(whatever). + * @param sz The size of @c buf. + * @param policy See @link MemoryPolicy @endlink . + */ + TMemoryBuffer(uint8_t* buf, uint32_t sz, MemoryPolicy policy = OBSERVE) { + if (buf == NULL && sz != 0) { + throw TTransportException(TTransportException::BAD_ARGS, + "TMemoryBuffer given null buffer with non-zero size."); + } + + switch (policy) { + case OBSERVE: + case TAKE_OWNERSHIP: + initCommon(buf, sz, policy == TAKE_OWNERSHIP, sz); + break; + case COPY: + initCommon(NULL, sz, true, 0); + this->write(buf, sz); + break; + default: + throw TTransportException(TTransportException::BAD_ARGS, + "Invalid MemoryPolicy for TMemoryBuffer"); + } + } + + ~TMemoryBuffer() { + if (owner_) { + std::free(buffer_); + } + } + + bool isOpen() { + return true; + } + + bool peek() { + return (rBase_ < wBase_); + } + + void open() {} + + void close() {} + + // TODO(dreiss): Make bufPtr const. + void getBuffer(uint8_t** bufPtr, uint32_t* sz) { + *bufPtr = rBase_; + *sz = wBase_ - rBase_; + } + + std::string getBufferAsString() { + if (buffer_ == NULL) { + return ""; + } + uint8_t* buf; + uint32_t sz; + getBuffer(&buf, &sz); + return std::string((char*)buf, (std::string::size_type)sz); + } + + void appendBufferToString(std::string& str) { + if (buffer_ == NULL) { + return; + } + uint8_t* buf; + uint32_t sz; + getBuffer(&buf, &sz); + str.append((char*)buf, sz); + } + + void resetBuffer(bool reset_capacity = false) { + if (reset_capacity) + { + assert(owner_); + + void* new_buffer = std::realloc(buffer_, defaultSize); + + if (new_buffer == NULL) { + throw TTransportException("Out of memory."); + } + + buffer_ = (uint8_t*) new_buffer; + bufferSize_ = defaultSize; + + wBound_ = buffer_ + bufferSize_; + } + + rBase_ = buffer_; + rBound_ = buffer_; + wBase_ = buffer_; + // It isn't safe to write into a buffer we don't own. + if (!owner_) { + wBound_ = wBase_; + bufferSize_ = 0; + } + } + + /// See constructor documentation. + void resetBuffer(uint8_t* buf, uint32_t sz, MemoryPolicy policy = OBSERVE) { + // Use a variant of the copy-and-swap trick for assignment operators. + // This is sub-optimal in terms of performance for two reasons: + // 1/ The constructing and swapping of the (small) values + // in the temporary object takes some time, and is not necessary. + // 2/ If policy == COPY, we allocate the new buffer before + // freeing the old one, precluding the possibility of + // reusing that memory. + // I doubt that either of these problems could be optimized away, + // but the second is probably no a common case, and the first is minor. + // I don't expect resetBuffer to be a common operation, so I'm willing to + // bite the performance bullet to make the method this simple. + + // Construct the new buffer. + TMemoryBuffer new_buffer(buf, sz, policy); + // Move it into ourself. + this->swap(new_buffer); + // Our old self gets destroyed. + } + + std::string readAsString(uint32_t len) { + std::string str; + (void)readAppendToString(str, len); + return str; + } + + uint32_t readAppendToString(std::string& str, uint32_t len); + + void readEnd() { + if (rBase_ == wBase_) { + resetBuffer(); + } + } + + uint32_t available_read() const { + // Remember, wBase_ is the real rBound_. + return wBase_ - rBase_; + } + + uint32_t available_write() const { + return wBound_ - wBase_; + } + + // Returns a pointer to where the client can write data to append to + // the TMemoryBuffer, and ensures the buffer is big enough to accomodate a + // write of the provided length. The returned pointer is very convenient for + // passing to read(), recv(), or similar. You must call wroteBytes() as soon + // as data is written or the buffer will not be aware that data has changed. + uint8_t* getWritePtr(uint32_t len) { + ensureCanWrite(len); + return wBase_; + } + + // Informs the buffer that the client has written 'len' bytes into storage + // that had been provided by getWritePtr(). + void wroteBytes(uint32_t len); + + protected: + void swap(TMemoryBuffer& that) { + using std::swap; + swap(buffer_, that.buffer_); + swap(bufferSize_, that.bufferSize_); + + swap(rBase_, that.rBase_); + swap(rBound_, that.rBound_); + swap(wBase_, that.wBase_); + swap(wBound_, that.wBound_); + + swap(owner_, that.owner_); + } + + // Make sure there's at least 'len' bytes available for writing. + void ensureCanWrite(uint32_t len); + + // Compute the position and available data for reading. + void computeRead(uint32_t len, uint8_t** out_start, uint32_t* out_give); + + uint32_t readSlow(uint8_t* buf, uint32_t len); + + void writeSlow(const uint8_t* buf, uint32_t len); + + const uint8_t* borrowSlow(uint8_t* buf, uint32_t* len); + + // Data buffer + uint8_t* buffer_; + + // Allocated buffer size + uint32_t bufferSize_; + + // Is this object the owner of the buffer? + bool owner_; + + // Don't forget to update constrctors, initCommon, and swap if + // you add new members. +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TBUFFERTRANSPORTS_H_ diff --git a/lib/cpp/src/transport/TFDTransport.cpp b/lib/cpp/src/transport/TFDTransport.cpp new file mode 100644 index 00000000..a042f8b7 --- /dev/null +++ b/lib/cpp/src/transport/TFDTransport.cpp @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include + +#include + +using namespace std; + +namespace apache { namespace thrift { namespace transport { + +void TFDTransport::close() { + if (!isOpen()) { + return; + } + + int rv = ::close(fd_); + int errno_copy = errno; + fd_ = -1; + // Have to check uncaught_exception because this is called in the destructor. + if (rv < 0 && !std::uncaught_exception()) { + throw TTransportException(TTransportException::UNKNOWN, + "TFDTransport::close()", + errno_copy); + } +} + +uint32_t TFDTransport::read(uint8_t* buf, uint32_t len) { + ssize_t rv = ::read(fd_, buf, len); + if (rv < 0) { + int errno_copy = errno; + throw TTransportException(TTransportException::UNKNOWN, + "TFDTransport::read()", + errno_copy); + } + return rv; +} + +void TFDTransport::write(const uint8_t* buf, uint32_t len) { + while (len > 0) { + ssize_t rv = ::write(fd_, buf, len); + + if (rv < 0) { + int errno_copy = errno; + throw TTransportException(TTransportException::UNKNOWN, + "TFDTransport::write()", + errno_copy); + } else if (rv == 0) { + throw TTransportException(TTransportException::END_OF_FILE, + "TFDTransport::write()"); + } + + buf += rv; + len -= rv; + } +} + +}}} // apache::thrift::transport diff --git a/lib/cpp/src/transport/TFDTransport.h b/lib/cpp/src/transport/TFDTransport.h new file mode 100644 index 00000000..bda5d82a --- /dev/null +++ b/lib/cpp/src/transport/TFDTransport.h @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TFDTRANSPORT_H_ +#define _THRIFT_TRANSPORT_TFDTRANSPORT_H_ 1 + +#include +#include + +#include "TTransport.h" +#include "TServerSocket.h" + +namespace apache { namespace thrift { namespace transport { + +/** + * Dead-simple wrapper around a file descriptor. + * + */ +class TFDTransport : public TTransport { + public: + enum ClosePolicy + { NO_CLOSE_ON_DESTROY = 0 + , CLOSE_ON_DESTROY = 1 + }; + + TFDTransport(int fd, ClosePolicy close_policy = NO_CLOSE_ON_DESTROY) + : fd_(fd) + , close_policy_(close_policy) + {} + + ~TFDTransport() { + if (close_policy_ == CLOSE_ON_DESTROY) { + close(); + } + } + + bool isOpen() { return fd_ >= 0; } + + void open() {} + + void close(); + + uint32_t read(uint8_t* buf, uint32_t len); + + void write(const uint8_t* buf, uint32_t len); + + void setFD(int fd) { fd_ = fd; } + int getFD() { return fd_; } + + protected: + int fd_; + ClosePolicy close_policy_; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TFDTRANSPORT_H_ diff --git a/lib/cpp/src/transport/TFileTransport.cpp b/lib/cpp/src/transport/TFileTransport.cpp new file mode 100644 index 00000000..f67b9e35 --- /dev/null +++ b/lib/cpp/src/transport/TFileTransport.cpp @@ -0,0 +1,953 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "TFileTransport.h" +#include "TTransportUtils.h" + +#include +#ifdef HAVE_SYS_TIME_H +#include +#else +#include +#endif +#include +#include +#include +#ifdef HAVE_STRINGS_H +#include +#endif +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace transport { + +using boost::shared_ptr; +using namespace std; +using namespace apache::thrift::protocol; + +#ifndef HAVE_CLOCK_GETTIME + +/** + * Fake clock_gettime for systems like darwin + * + */ +#define CLOCK_REALTIME 0 +static int clock_gettime(int clk_id /*ignored*/, struct timespec *tp) { + struct timeval now; + + int rv = gettimeofday(&now, NULL); + if (rv != 0) { + return rv; + } + + tp->tv_sec = now.tv_sec; + tp->tv_nsec = now.tv_usec * 1000; + return 0; +} +#endif + +TFileTransport::TFileTransport(string path, bool readOnly) + : readState_() + , readBuff_(NULL) + , currentEvent_(NULL) + , readBuffSize_(DEFAULT_READ_BUFF_SIZE) + , readTimeout_(NO_TAIL_READ_TIMEOUT) + , chunkSize_(DEFAULT_CHUNK_SIZE) + , eventBufferSize_(DEFAULT_EVENT_BUFFER_SIZE) + , flushMaxUs_(DEFAULT_FLUSH_MAX_US) + , flushMaxBytes_(DEFAULT_FLUSH_MAX_BYTES) + , maxEventSize_(DEFAULT_MAX_EVENT_SIZE) + , maxCorruptedEvents_(DEFAULT_MAX_CORRUPTED_EVENTS) + , eofSleepTime_(DEFAULT_EOF_SLEEP_TIME_US) + , corruptedEventSleepTime_(DEFAULT_CORRUPTED_SLEEP_TIME_US) + , writerThreadId_(0) + , dequeueBuffer_(NULL) + , enqueueBuffer_(NULL) + , closing_(false) + , forceFlush_(false) + , filename_(path) + , fd_(0) + , bufferAndThreadInitialized_(false) + , offset_(0) + , lastBadChunk_(0) + , numCorruptedEventsInChunk_(0) + , readOnly_(readOnly) +{ + // initialize all the condition vars/mutexes + pthread_mutex_init(&mutex_, NULL); + pthread_cond_init(¬Full_, NULL); + pthread_cond_init(¬Empty_, NULL); + pthread_cond_init(&flushed_, NULL); + + openLogFile(); +} + +void TFileTransport::resetOutputFile(int fd, string filename, int64_t offset) { + filename_ = filename; + offset_ = offset; + + // check if current file is still open + if (fd_ > 0) { + // flush any events in the queue + flush(); + GlobalOutput.printf("error, current file (%s) not closed", filename_.c_str()); + if (-1 == ::close(fd_)) { + int errno_copy = errno; + GlobalOutput.perror("TFileTransport: resetOutputFile() ::close() ", errno_copy); + throw TTransportException(TTransportException::UNKNOWN, "TFileTransport: error in file close", errno_copy); + } + } + + if (fd) { + fd_ = fd; + } else { + // open file if the input fd is 0 + openLogFile(); + } +} + + +TFileTransport::~TFileTransport() { + // flush the buffer if a writer thread is active + if (writerThreadId_ > 0) { + // reduce the flush timeout so that closing is quicker + setFlushMaxUs(300*1000); + + // flush output buffer + flush(); + + // set state to closing + closing_ = true; + + // TODO: make sure event queue is empty + // currently only the write buffer is flushed + // we dont actually wait until the queue is empty. This shouldn't be a big + // deal in the common case because writing is quick + + pthread_join(writerThreadId_, NULL); + writerThreadId_ = 0; + } + + if (dequeueBuffer_) { + delete dequeueBuffer_; + dequeueBuffer_ = NULL; + } + + if (enqueueBuffer_) { + delete enqueueBuffer_; + enqueueBuffer_ = NULL; + } + + if (readBuff_) { + delete[] readBuff_; + readBuff_ = NULL; + } + + if (currentEvent_) { + delete currentEvent_; + currentEvent_ = NULL; + } + + // close logfile + if (fd_ > 0) { + if(-1 == ::close(fd_)) { + GlobalOutput.perror("TFileTransport: ~TFileTransport() ::close() ", errno); + } + } +} + +bool TFileTransport::initBufferAndWriteThread() { + if (bufferAndThreadInitialized_) { + T_ERROR("Trying to double-init TFileTransport"); + return false; + } + + if (writerThreadId_ == 0) { + if (pthread_create(&writerThreadId_, NULL, startWriterThread, (void *)this) != 0) { + T_ERROR("Could not create writer thread"); + return false; + } + } + + dequeueBuffer_ = new TFileTransportBuffer(eventBufferSize_); + enqueueBuffer_ = new TFileTransportBuffer(eventBufferSize_); + bufferAndThreadInitialized_ = true; + + return true; +} + +void TFileTransport::write(const uint8_t* buf, uint32_t len) { + if (readOnly_) { + throw TTransportException("TFileTransport: attempting to write to file opened readonly"); + } + + enqueueEvent(buf, len, false); +} + +void TFileTransport::enqueueEvent(const uint8_t* buf, uint32_t eventLen, bool blockUntilFlush) { + // can't enqueue more events if file is going to close + if (closing_) { + return; + } + + // make sure that event size is valid + if ( (maxEventSize_ > 0) && (eventLen > maxEventSize_) ) { + T_ERROR("msg size is greater than max event size: %u > %u\n", eventLen, maxEventSize_); + return; + } + + if (eventLen == 0) { + T_ERROR("cannot enqueue an empty event"); + return; + } + + eventInfo* toEnqueue = new eventInfo(); + toEnqueue->eventBuff_ = (uint8_t *)std::malloc((sizeof(uint8_t) * eventLen) + 4); + // first 4 bytes is the event length + memcpy(toEnqueue->eventBuff_, (void*)(&eventLen), 4); + // actual event contents + memcpy(toEnqueue->eventBuff_ + 4, buf, eventLen); + toEnqueue->eventSize_ = eventLen + 4; + + // lock mutex + pthread_mutex_lock(&mutex_); + + // make sure that enqueue buffer is initialized and writer thread is running + if (!bufferAndThreadInitialized_) { + if (!initBufferAndWriteThread()) { + delete toEnqueue; + pthread_mutex_unlock(&mutex_); + return; + } + } + + // Can't enqueue while buffer is full + while (enqueueBuffer_->isFull()) { + pthread_cond_wait(¬Full_, &mutex_); + } + + // add to the buffer + if (!enqueueBuffer_->addEvent(toEnqueue)) { + delete toEnqueue; + pthread_mutex_unlock(&mutex_); + return; + } + + // signal anybody who's waiting for the buffer to be non-empty + pthread_cond_signal(¬Empty_); + + if (blockUntilFlush) { + pthread_cond_wait(&flushed_, &mutex_); + } + + // this really should be a loop where it makes sure it got flushed + // because condition variables can get triggered by the os for no reason + // it is probably a non-factor for the time being + pthread_mutex_unlock(&mutex_); +} + +bool TFileTransport::swapEventBuffers(struct timespec* deadline) { + pthread_mutex_lock(&mutex_); + if (deadline != NULL) { + // if we were handed a deadline time struct, do a timed wait + pthread_cond_timedwait(¬Empty_, &mutex_, deadline); + } else { + // just wait until the buffer gets an item + pthread_cond_wait(¬Empty_, &mutex_); + } + + bool swapped = false; + + // could be empty if we timed out + if (!enqueueBuffer_->isEmpty()) { + TFileTransportBuffer *temp = enqueueBuffer_; + enqueueBuffer_ = dequeueBuffer_; + dequeueBuffer_ = temp; + + swapped = true; + } + + // unlock the mutex and signal if required + pthread_mutex_unlock(&mutex_); + + if (swapped) { + pthread_cond_signal(¬Full_); + } + + return swapped; +} + + +void TFileTransport::writerThread() { + // open file if it is not open + if(!fd_) { + openLogFile(); + } + + // set the offset to the correct value (EOF) + try { + seekToEnd(); + } catch (TException &te) { + } + + // throw away any partial events + offset_ += readState_.lastDispatchPtr_; + ftruncate(fd_, offset_); + readState_.resetAllValues(); + + // Figure out the next time by which a flush must take place + + struct timespec ts_next_flush; + getNextFlushTime(&ts_next_flush); + uint32_t unflushed = 0; + + while(1) { + // this will only be true when the destructor is being invoked + if(closing_) { + // empty out both the buffers + if (enqueueBuffer_->isEmpty() && dequeueBuffer_->isEmpty()) { + if (-1 == ::close(fd_)) { + int errno_copy = errno; + GlobalOutput.perror("TFileTransport: writerThread() ::close() ", errno_copy); + throw TTransportException(TTransportException::UNKNOWN, "TFileTransport: error in file close", errno_copy); + } + // just be safe and sync to disk + fsync(fd_); + fd_ = 0; + pthread_exit(NULL); + return; + } + } + + if (swapEventBuffers(&ts_next_flush)) { + eventInfo* outEvent; + while (NULL != (outEvent = dequeueBuffer_->getNext())) { + if (!outEvent) { + T_DEBUG_L(1, "Got an empty event"); + return; + } + + // sanity check on event + if ((maxEventSize_ > 0) && (outEvent->eventSize_ > maxEventSize_)) { + T_ERROR("msg size is greater than max event size: %u > %u\n", outEvent->eventSize_, maxEventSize_); + continue; + } + + // If chunking is required, then make sure that msg does not cross chunk boundary + if ((outEvent->eventSize_ > 0) && (chunkSize_ != 0)) { + + // event size must be less than chunk size + if(outEvent->eventSize_ > chunkSize_) { + T_ERROR("TFileTransport: event size(%u) is greater than chunk size(%u): skipping event", + outEvent->eventSize_, chunkSize_); + continue; + } + + int64_t chunk1 = offset_/chunkSize_; + int64_t chunk2 = (offset_ + outEvent->eventSize_ - 1)/chunkSize_; + + // if adding this event will cross a chunk boundary, pad the chunk with zeros + if (chunk1 != chunk2) { + // refetch the offset to keep in sync + offset_ = lseek(fd_, 0, SEEK_CUR); + int32_t padding = (int32_t)((offset_/chunkSize_ + 1)*chunkSize_ - offset_); + + uint8_t zeros[padding]; + bzero(zeros, padding); + if (-1 == ::write(fd_, zeros, padding)) { + int errno_copy = errno; + GlobalOutput.perror("TFileTransport: writerThread() error while padding zeros ", errno_copy); + throw TTransportException(TTransportException::UNKNOWN, "TFileTransport: error while padding zeros", errno_copy); + } + unflushed += padding; + offset_ += padding; + } + } + + // write the dequeued event to the file + if (outEvent->eventSize_ > 0) { + if (-1 == ::write(fd_, outEvent->eventBuff_, outEvent->eventSize_)) { + int errno_copy = errno; + GlobalOutput.perror("TFileTransport: error while writing event ", errno_copy); + throw TTransportException(TTransportException::UNKNOWN, "TFileTransport: error while writing event", errno_copy); + } + + unflushed += outEvent->eventSize_; + offset_ += outEvent->eventSize_; + } + } + dequeueBuffer_->reset(); + } + + bool flushTimeElapsed = false; + struct timespec current_time; + clock_gettime(CLOCK_REALTIME, ¤t_time); + + if (current_time.tv_sec > ts_next_flush.tv_sec || + (current_time.tv_sec == ts_next_flush.tv_sec && current_time.tv_nsec > ts_next_flush.tv_nsec)) { + flushTimeElapsed = true; + getNextFlushTime(&ts_next_flush); + } + + // couple of cases from which a flush could be triggered + if ((flushTimeElapsed && unflushed > 0) || + unflushed > flushMaxBytes_ || + forceFlush_) { + + // sync (force flush) file to disk + fsync(fd_); + unflushed = 0; + + // notify anybody waiting for flush completion + forceFlush_ = false; + pthread_cond_broadcast(&flushed_); + } + } +} + +void TFileTransport::flush() { + // file must be open for writing for any flushing to take place + if (writerThreadId_ <= 0) { + return; + } + // wait for flush to take place + pthread_mutex_lock(&mutex_); + + forceFlush_ = true; + + while (forceFlush_) { + pthread_cond_wait(&flushed_, &mutex_); + } + + pthread_mutex_unlock(&mutex_); +} + + +uint32_t TFileTransport::readAll(uint8_t* buf, uint32_t len) { + uint32_t have = 0; + uint32_t get = 0; + + while (have < len) { + get = read(buf+have, len-have); + if (get <= 0) { + throw TEOFException(); + } + have += get; + } + + return have; +} + +uint32_t TFileTransport::read(uint8_t* buf, uint32_t len) { + // check if there an event is ready to be read + if (!currentEvent_) { + currentEvent_ = readEvent(); + } + + // did not manage to read an event from the file. This could have happened + // if the timeout expired or there was some other error + if (!currentEvent_) { + return 0; + } + + // read as much of the current event as possible + int32_t remaining = currentEvent_->eventSize_ - currentEvent_->eventBuffPos_; + if (remaining <= (int32_t)len) { + // copy over anything thats remaining + if (remaining > 0) { + memcpy(buf, + currentEvent_->eventBuff_ + currentEvent_->eventBuffPos_, + remaining); + } + delete(currentEvent_); + currentEvent_ = NULL; + return remaining; + } + + // read as much as possible + memcpy(buf, currentEvent_->eventBuff_ + currentEvent_->eventBuffPos_, len); + currentEvent_->eventBuffPos_ += len; + return len; +} + +eventInfo* TFileTransport::readEvent() { + int readTries = 0; + + if (!readBuff_) { + readBuff_ = new uint8_t[readBuffSize_]; + } + + while (1) { + // read from the file if read buffer is exhausted + if (readState_.bufferPtr_ == readState_.bufferLen_) { + // advance the offset pointer + offset_ += readState_.bufferLen_; + readState_.bufferLen_ = ::read(fd_, readBuff_, readBuffSize_); + // if (readState_.bufferLen_) { + // T_DEBUG_L(1, "Amount read: %u (offset: %lu)", readState_.bufferLen_, offset_); + // } + readState_.bufferPtr_ = 0; + readState_.lastDispatchPtr_ = 0; + + // read error + if (readState_.bufferLen_ == -1) { + readState_.resetAllValues(); + GlobalOutput("TFileTransport: error while reading from file"); + throw TTransportException("TFileTransport: error while reading from file"); + } else if (readState_.bufferLen_ == 0) { // EOF + // wait indefinitely if there is no timeout + if (readTimeout_ == TAIL_READ_TIMEOUT) { + usleep(eofSleepTime_); + continue; + } else if (readTimeout_ == NO_TAIL_READ_TIMEOUT) { + // reset state + readState_.resetState(0); + return NULL; + } else if (readTimeout_ > 0) { + // timeout already expired once + if (readTries > 0) { + readState_.resetState(0); + return NULL; + } else { + usleep(readTimeout_ * 1000); + readTries++; + continue; + } + } + } + } + + readTries = 0; + + // attempt to read an event from the buffer + while(readState_.bufferPtr_ < readState_.bufferLen_) { + if (readState_.readingSize_) { + if(readState_.eventSizeBuffPos_ == 0) { + if ( (offset_ + readState_.bufferPtr_)/chunkSize_ != + ((offset_ + readState_.bufferPtr_ + 3)/chunkSize_)) { + // skip one byte towards chunk boundary + // T_DEBUG_L(1, "Skipping a byte"); + readState_.bufferPtr_++; + continue; + } + } + + readState_.eventSizeBuff_[readState_.eventSizeBuffPos_++] = + readBuff_[readState_.bufferPtr_++]; + if (readState_.eventSizeBuffPos_ == 4) { + // 0 length event indicates padding + if (*((uint32_t *)(readState_.eventSizeBuff_)) == 0) { + // T_DEBUG_L(1, "Got padding"); + readState_.resetState(readState_.lastDispatchPtr_); + continue; + } + // got a valid event + readState_.readingSize_ = false; + if (readState_.event_) { + delete(readState_.event_); + } + readState_.event_ = new eventInfo(); + readState_.event_->eventSize_ = *((uint32_t *)(readState_.eventSizeBuff_)); + + // check if the event is corrupted and perform recovery if required + if (isEventCorrupted()) { + performRecovery(); + // start from the top + break; + } + } + } else { + if (!readState_.event_->eventBuff_) { + readState_.event_->eventBuff_ = new uint8_t[readState_.event_->eventSize_]; + readState_.event_->eventBuffPos_ = 0; + } + // take either the entire event or the remaining bytes in the buffer + int reclaimBuffer = min((uint32_t)(readState_.bufferLen_ - readState_.bufferPtr_), + readState_.event_->eventSize_ - readState_.event_->eventBuffPos_); + + // copy data from read buffer into event buffer + memcpy(readState_.event_->eventBuff_ + readState_.event_->eventBuffPos_, + readBuff_ + readState_.bufferPtr_, + reclaimBuffer); + + // increment position ptrs + readState_.event_->eventBuffPos_ += reclaimBuffer; + readState_.bufferPtr_ += reclaimBuffer; + + // check if the event has been read in full + if (readState_.event_->eventBuffPos_ == readState_.event_->eventSize_) { + // set the completed event to the current event + eventInfo* completeEvent = readState_.event_; + completeEvent->eventBuffPos_ = 0; + + readState_.event_ = NULL; + readState_.resetState(readState_.bufferPtr_); + + // exit criteria + return completeEvent; + } + } + } + + } +} + +bool TFileTransport::isEventCorrupted() { + // an error is triggered if: + if ( (maxEventSize_ > 0) && (readState_.event_->eventSize_ > maxEventSize_)) { + // 1. Event size is larger than user-speficied max-event size + T_ERROR("Read corrupt event. Event size(%u) greater than max event size (%u)", + readState_.event_->eventSize_, maxEventSize_); + return true; + } else if (readState_.event_->eventSize_ > chunkSize_) { + // 2. Event size is larger than chunk size + T_ERROR("Read corrupt event. Event size(%u) greater than chunk size (%u)", + readState_.event_->eventSize_, chunkSize_); + return true; + } else if( ((offset_ + readState_.bufferPtr_ - 4)/chunkSize_) != + ((offset_ + readState_.bufferPtr_ + readState_.event_->eventSize_ - 1)/chunkSize_) ) { + // 3. size indicates that event crosses chunk boundary + T_ERROR("Read corrupt event. Event crosses chunk boundary. Event size:%u Offset:%ld", + readState_.event_->eventSize_, offset_ + readState_.bufferPtr_ + 4); + return true; + } + + return false; +} + +void TFileTransport::performRecovery() { + // perform some kickass recovery + uint32_t curChunk = getCurChunk(); + if (lastBadChunk_ == curChunk) { + numCorruptedEventsInChunk_++; + } else { + lastBadChunk_ = curChunk; + numCorruptedEventsInChunk_ = 1; + } + + if (numCorruptedEventsInChunk_ < maxCorruptedEvents_) { + // maybe there was an error in reading the file from disk + // seek to the beginning of chunk and try again + seekToChunk(curChunk); + } else { + + // just skip ahead to the next chunk if we not already at the last chunk + if (curChunk != (getNumChunks() - 1)) { + seekToChunk(curChunk + 1); + } else if (readTimeout_ == TAIL_READ_TIMEOUT) { + // if tailing the file, wait until there is enough data to start + // the next chunk + while(curChunk == (getNumChunks() - 1)) { + usleep(DEFAULT_CORRUPTED_SLEEP_TIME_US); + } + seekToChunk(curChunk + 1); + } else { + // pretty hosed at this stage, rewind the file back to the last successful + // point and punt on the error + readState_.resetState(readState_.lastDispatchPtr_); + currentEvent_ = NULL; + char errorMsg[1024]; + sprintf(errorMsg, "TFileTransport: log file corrupted at offset: %lu", + offset_ + readState_.lastDispatchPtr_); + GlobalOutput(errorMsg); + throw TTransportException(errorMsg); + } + } + +} + +void TFileTransport::seekToChunk(int32_t chunk) { + if (fd_ <= 0) { + throw TTransportException("File not open"); + } + + int32_t numChunks = getNumChunks(); + + // file is empty, seeking to chunk is pointless + if (numChunks == 0) { + return; + } + + // negative indicates reverse seek (from the end) + if (chunk < 0) { + chunk += numChunks; + } + + // too large a value for reverse seek, just seek to beginning + if (chunk < 0) { + T_DEBUG("Incorrect value for reverse seek. Seeking to beginning...", chunk) + chunk = 0; + } + + // cannot seek past EOF + bool seekToEnd = false; + uint32_t minEndOffset = 0; + if (chunk >= numChunks) { + T_DEBUG("Trying to seek past EOF. Seeking to EOF instead..."); + seekToEnd = true; + chunk = numChunks - 1; + // this is the min offset to process events till + minEndOffset = lseek(fd_, 0, SEEK_END); + } + + off_t newOffset = off_t(chunk) * chunkSize_; + offset_ = lseek(fd_, newOffset, SEEK_SET); + readState_.resetAllValues(); + currentEvent_ = NULL; + if (offset_ == -1) { + GlobalOutput("TFileTransport: lseek error in seekToChunk"); + throw TTransportException("TFileTransport: lseek error in seekToChunk"); + } + + // seek to EOF if user wanted to go to last chunk + if (seekToEnd) { + uint32_t oldReadTimeout = getReadTimeout(); + setReadTimeout(NO_TAIL_READ_TIMEOUT); + // keep on reading unti the last event at point of seekChunk call + while (readEvent() && ((offset_ + readState_.bufferPtr_) < minEndOffset)) {}; + setReadTimeout(oldReadTimeout); + } + +} + +void TFileTransport::seekToEnd() { + seekToChunk(getNumChunks()); +} + +uint32_t TFileTransport::getNumChunks() { + if (fd_ <= 0) { + return 0; + } + + struct stat f_info; + int rv = fstat(fd_, &f_info); + + if (rv < 0) { + int errno_copy = errno; + throw TTransportException(TTransportException::UNKNOWN, + "TFileTransport::getNumChunks() (fstat)", + errno_copy); + } + + if (f_info.st_size > 0) { + return ((f_info.st_size)/chunkSize_) + 1; + } + + // empty file has no chunks + return 0; +} + +uint32_t TFileTransport::getCurChunk() { + return offset_/chunkSize_; +} + +// Utility Functions +void TFileTransport::openLogFile() { + mode_t mode = readOnly_ ? S_IRUSR | S_IRGRP | S_IROTH : S_IRUSR | S_IWUSR| S_IRGRP | S_IROTH; + int flags = readOnly_ ? O_RDONLY : O_RDWR | O_CREAT | O_APPEND; + fd_ = ::open(filename_.c_str(), flags, mode); + offset_ = 0; + + // make sure open call was successful + if(fd_ == -1) { + int errno_copy = errno; + GlobalOutput.perror("TFileTransport: openLogFile() ::open() file: " + filename_, errno_copy); + throw TTransportException(TTransportException::NOT_OPEN, filename_, errno_copy); + } + +} + +void TFileTransport::getNextFlushTime(struct timespec* ts_next_flush) { + clock_gettime(CLOCK_REALTIME, ts_next_flush); + ts_next_flush->tv_nsec += (flushMaxUs_ % 1000000) * 1000; + if (ts_next_flush->tv_nsec > 1000000000) { + ts_next_flush->tv_nsec -= 1000000000; + ts_next_flush->tv_sec += 1; + } + ts_next_flush->tv_sec += flushMaxUs_ / 1000000; +} + +TFileTransportBuffer::TFileTransportBuffer(uint32_t size) + : bufferMode_(WRITE) + , writePoint_(0) + , readPoint_(0) + , size_(size) +{ + buffer_ = new eventInfo*[size]; +} + +TFileTransportBuffer::~TFileTransportBuffer() { + if (buffer_) { + for (uint32_t i = 0; i < writePoint_; i++) { + delete buffer_[i]; + } + delete[] buffer_; + buffer_ = NULL; + } +} + +bool TFileTransportBuffer::addEvent(eventInfo *event) { + if (bufferMode_ == READ) { + GlobalOutput("Trying to write to a buffer in read mode"); + } + if (writePoint_ < size_) { + buffer_[writePoint_++] = event; + return true; + } else { + // buffer is full + return false; + } +} + +eventInfo* TFileTransportBuffer::getNext() { + if (bufferMode_ == WRITE) { + bufferMode_ = READ; + } + if (readPoint_ < writePoint_) { + return buffer_[readPoint_++]; + } else { + // no more entries + return NULL; + } +} + +void TFileTransportBuffer::reset() { + if (bufferMode_ == WRITE || writePoint_ > readPoint_) { + T_DEBUG("Resetting a buffer with unread entries"); + } + // Clean up the old entries + for (uint32_t i = 0; i < writePoint_; i++) { + delete buffer_[i]; + } + bufferMode_ = WRITE; + writePoint_ = 0; + readPoint_ = 0; +} + +bool TFileTransportBuffer::isFull() { + return writePoint_ == size_; +} + +bool TFileTransportBuffer::isEmpty() { + return writePoint_ == 0; +} + +TFileProcessor::TFileProcessor(shared_ptr processor, + shared_ptr protocolFactory, + shared_ptr inputTransport): + processor_(processor), + inputProtocolFactory_(protocolFactory), + outputProtocolFactory_(protocolFactory), + inputTransport_(inputTransport) { + + // default the output transport to a null transport (common case) + outputTransport_ = shared_ptr(new TNullTransport()); +} + +TFileProcessor::TFileProcessor(shared_ptr processor, + shared_ptr inputProtocolFactory, + shared_ptr outputProtocolFactory, + shared_ptr inputTransport): + processor_(processor), + inputProtocolFactory_(inputProtocolFactory), + outputProtocolFactory_(outputProtocolFactory), + inputTransport_(inputTransport) { + + // default the output transport to a null transport (common case) + outputTransport_ = shared_ptr(new TNullTransport()); +} + +TFileProcessor::TFileProcessor(shared_ptr processor, + shared_ptr protocolFactory, + shared_ptr inputTransport, + shared_ptr outputTransport): + processor_(processor), + inputProtocolFactory_(protocolFactory), + outputProtocolFactory_(protocolFactory), + inputTransport_(inputTransport), + outputTransport_(outputTransport) {}; + +void TFileProcessor::process(uint32_t numEvents, bool tail) { + shared_ptr inputProtocol = inputProtocolFactory_->getProtocol(inputTransport_); + shared_ptr outputProtocol = outputProtocolFactory_->getProtocol(outputTransport_); + + // set the read timeout to 0 if tailing is required + int32_t oldReadTimeout = inputTransport_->getReadTimeout(); + if (tail) { + // save old read timeout so it can be restored + inputTransport_->setReadTimeout(TFileTransport::TAIL_READ_TIMEOUT); + } + + uint32_t numProcessed = 0; + while(1) { + // bad form to use exceptions for flow control but there is really + // no other way around it + try { + processor_->process(inputProtocol, outputProtocol); + numProcessed++; + if ( (numEvents > 0) && (numProcessed == numEvents)) { + return; + } + } catch (TEOFException& teof) { + if (!tail) { + break; + } + } catch (TException &te) { + cerr << te.what() << endl; + break; + } + } + + // restore old read timeout + if (tail) { + inputTransport_->setReadTimeout(oldReadTimeout); + } + +} + +void TFileProcessor::processChunk() { + shared_ptr inputProtocol = inputProtocolFactory_->getProtocol(inputTransport_); + shared_ptr outputProtocol = outputProtocolFactory_->getProtocol(outputTransport_); + + uint32_t curChunk = inputTransport_->getCurChunk(); + + while(1) { + // bad form to use exceptions for flow control but there is really + // no other way around it + try { + processor_->process(inputProtocol, outputProtocol); + if (curChunk != inputTransport_->getCurChunk()) { + break; + } + } catch (TEOFException& teof) { + break; + } catch (TException &te) { + cerr << te.what() << endl; + break; + } + } +} + +}}} // apache::thrift::transport diff --git a/lib/cpp/src/transport/TFileTransport.h b/lib/cpp/src/transport/TFileTransport.h new file mode 100644 index 00000000..fbaf2cd0 --- /dev/null +++ b/lib/cpp/src/transport/TFileTransport.h @@ -0,0 +1,440 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TFILETRANSPORT_H_ +#define _THRIFT_TRANSPORT_TFILETRANSPORT_H_ 1 + +#include "TTransport.h" +#include "Thrift.h" +#include "TProcessor.h" + +#include +#include + +#include + +namespace apache { namespace thrift { namespace transport { + +using apache::thrift::TProcessor; +using apache::thrift::protocol::TProtocolFactory; + +// Data pertaining to a single event +typedef struct eventInfo { + uint8_t* eventBuff_; + uint32_t eventSize_; + uint32_t eventBuffPos_; + + eventInfo():eventBuff_(NULL), eventSize_(0), eventBuffPos_(0){}; + ~eventInfo() { + if (eventBuff_) { + delete[] eventBuff_; + } + } +} eventInfo; + +// information about current read state +typedef struct readState { + eventInfo* event_; + + // keep track of event size + uint8_t eventSizeBuff_[4]; + uint8_t eventSizeBuffPos_; + bool readingSize_; + + // read buffer variables + int32_t bufferPtr_; + int32_t bufferLen_; + + // last successful dispatch point + int32_t lastDispatchPtr_; + + void resetState(uint32_t lastDispatchPtr) { + readingSize_ = true; + eventSizeBuffPos_ = 0; + lastDispatchPtr_ = lastDispatchPtr; + } + + void resetAllValues() { + resetState(0); + bufferPtr_ = 0; + bufferLen_ = 0; + if (event_) { + delete(event_); + } + event_ = 0; + } + + readState() { + event_ = 0; + resetAllValues(); + } + + ~readState() { + if (event_) { + delete(event_); + } + } + +} readState; + +/** + * TFileTransportBuffer - buffer class used by TFileTransport for queueing up events + * to be written to disk. Should be used in the following way: + * 1) Buffer created + * 2) Buffer written to (addEvent) + * 3) Buffer read from (getNext) + * 4) Buffer reset (reset) + * 5) Go back to 2, or destroy buffer + * + * The buffer should never be written to after it is read from, unless it is reset first. + * Note: The above rules are enforced mainly for debugging its sole client TFileTransport + * which uses the buffer in this way. + * + */ +class TFileTransportBuffer { + public: + TFileTransportBuffer(uint32_t size); + ~TFileTransportBuffer(); + + bool addEvent(eventInfo *event); + eventInfo* getNext(); + void reset(); + bool isFull(); + bool isEmpty(); + + private: + TFileTransportBuffer(); // should not be used + + enum mode { + WRITE, + READ + }; + mode bufferMode_; + + uint32_t writePoint_; + uint32_t readPoint_; + uint32_t size_; + eventInfo** buffer_; +}; + +/** + * Abstract interface for transports used to read files + */ +class TFileReaderTransport : virtual public TTransport { + public: + virtual int32_t getReadTimeout() = 0; + virtual void setReadTimeout(int32_t readTimeout) = 0; + + virtual uint32_t getNumChunks() = 0; + virtual uint32_t getCurChunk() = 0; + virtual void seekToChunk(int32_t chunk) = 0; + virtual void seekToEnd() = 0; +}; + +/** + * Abstract interface for transports used to write files + */ +class TFileWriterTransport : virtual public TTransport { + public: + virtual uint32_t getChunkSize() = 0; + virtual void setChunkSize(uint32_t chunkSize) = 0; +}; + +/** + * File implementation of a transport. Reads and writes are done to a + * file on disk. + * + */ +class TFileTransport : public TFileReaderTransport, + public TFileWriterTransport { + public: + TFileTransport(std::string path, bool readOnly=false); + ~TFileTransport(); + + // TODO: what is the correct behaviour for this? + // the log file is generally always open + bool isOpen() { + return true; + } + + void write(const uint8_t* buf, uint32_t len); + void flush(); + + uint32_t readAll(uint8_t* buf, uint32_t len); + uint32_t read(uint8_t* buf, uint32_t len); + + // log-file specific functions + void seekToChunk(int32_t chunk); + void seekToEnd(); + uint32_t getNumChunks(); + uint32_t getCurChunk(); + + // for changing the output file + void resetOutputFile(int fd, std::string filename, int64_t offset); + + // Setter/Getter functions for user-controllable options + void setReadBuffSize(uint32_t readBuffSize) { + if (readBuffSize) { + readBuffSize_ = readBuffSize; + } + } + uint32_t getReadBuffSize() { + return readBuffSize_; + } + + static const int32_t TAIL_READ_TIMEOUT = -1; + static const int32_t NO_TAIL_READ_TIMEOUT = 0; + void setReadTimeout(int32_t readTimeout) { + readTimeout_ = readTimeout; + } + int32_t getReadTimeout() { + return readTimeout_; + } + + void setChunkSize(uint32_t chunkSize) { + if (chunkSize) { + chunkSize_ = chunkSize; + } + } + uint32_t getChunkSize() { + return chunkSize_; + } + + void setEventBufferSize(uint32_t bufferSize) { + if (bufferAndThreadInitialized_) { + GlobalOutput("Cannot change the buffer size after writer thread started"); + return; + } + eventBufferSize_ = bufferSize; + } + + uint32_t getEventBufferSize() { + return eventBufferSize_; + } + + void setFlushMaxUs(uint32_t flushMaxUs) { + if (flushMaxUs) { + flushMaxUs_ = flushMaxUs; + } + } + uint32_t getFlushMaxUs() { + return flushMaxUs_; + } + + void setFlushMaxBytes(uint32_t flushMaxBytes) { + if (flushMaxBytes) { + flushMaxBytes_ = flushMaxBytes; + } + } + uint32_t getFlushMaxBytes() { + return flushMaxBytes_; + } + + void setMaxEventSize(uint32_t maxEventSize) { + maxEventSize_ = maxEventSize; + } + uint32_t getMaxEventSize() { + return maxEventSize_; + } + + void setMaxCorruptedEvents(uint32_t maxCorruptedEvents) { + maxCorruptedEvents_ = maxCorruptedEvents; + } + uint32_t getMaxCorruptedEvents() { + return maxCorruptedEvents_; + } + + void setEofSleepTimeUs(uint32_t eofSleepTime) { + if (eofSleepTime) { + eofSleepTime_ = eofSleepTime; + } + } + uint32_t getEofSleepTimeUs() { + return eofSleepTime_; + } + + private: + // helper functions for writing to a file + void enqueueEvent(const uint8_t* buf, uint32_t eventLen, bool blockUntilFlush); + bool swapEventBuffers(struct timespec* deadline); + bool initBufferAndWriteThread(); + + // control for writer thread + static void* startWriterThread(void* ptr) { + (((TFileTransport*)ptr)->writerThread()); + return 0; + } + void writerThread(); + + // helper functions for reading from a file + eventInfo* readEvent(); + + // event corruption-related functions + bool isEventCorrupted(); + void performRecovery(); + + // Utility functions + void openLogFile(); + void getNextFlushTime(struct timespec* ts_next_flush); + + // Class variables + readState readState_; + uint8_t* readBuff_; + eventInfo* currentEvent_; + + uint32_t readBuffSize_; + static const uint32_t DEFAULT_READ_BUFF_SIZE = 1 * 1024 * 1024; + + int32_t readTimeout_; + static const int32_t DEFAULT_READ_TIMEOUT_MS = 200; + + // size of chunks that file will be split up into + uint32_t chunkSize_; + static const uint32_t DEFAULT_CHUNK_SIZE = 16 * 1024 * 1024; + + // size of event buffers + uint32_t eventBufferSize_; + static const uint32_t DEFAULT_EVENT_BUFFER_SIZE = 10000; + + // max number of microseconds that can pass without flushing + uint32_t flushMaxUs_; + static const uint32_t DEFAULT_FLUSH_MAX_US = 3000000; + + // max number of bytes that can be written without flushing + uint32_t flushMaxBytes_; + static const uint32_t DEFAULT_FLUSH_MAX_BYTES = 1000 * 1024; + + // max event size + uint32_t maxEventSize_; + static const uint32_t DEFAULT_MAX_EVENT_SIZE = 0; + + // max number of corrupted events per chunk + uint32_t maxCorruptedEvents_; + static const uint32_t DEFAULT_MAX_CORRUPTED_EVENTS = 0; + + // sleep duration when EOF is hit + uint32_t eofSleepTime_; + static const uint32_t DEFAULT_EOF_SLEEP_TIME_US = 500 * 1000; + + // sleep duration when a corrupted event is encountered + uint32_t corruptedEventSleepTime_; + static const uint32_t DEFAULT_CORRUPTED_SLEEP_TIME_US = 1 * 1000 * 1000; + + // writer thread id + pthread_t writerThreadId_; + + // buffers to hold data before it is flushed. Each element of the buffer stores a msg that + // needs to be written to the file. The buffers are swapped by the writer thread. + TFileTransportBuffer *dequeueBuffer_; + TFileTransportBuffer *enqueueBuffer_; + + // conditions used to block when the buffer is full or empty + pthread_cond_t notFull_, notEmpty_; + volatile bool closing_; + + // To keep track of whether the buffer has been flushed + pthread_cond_t flushed_; + volatile bool forceFlush_; + + // Mutex that is grabbed when enqueueing and swapping the read/write buffers + pthread_mutex_t mutex_; + + // File information + std::string filename_; + int fd_; + + // Whether the writer thread and buffers have been initialized + bool bufferAndThreadInitialized_; + + // Offset within the file + off_t offset_; + + // event corruption information + uint32_t lastBadChunk_; + uint32_t numCorruptedEventsInChunk_; + + bool readOnly_; +}; + +// Exception thrown when EOF is hit +class TEOFException : public TTransportException { + public: + TEOFException(): + TTransportException(TTransportException::END_OF_FILE) {}; +}; + + +// wrapper class to process events from a file containing thrift events +class TFileProcessor { + public: + /** + * Constructor that defaults output transport to null transport + * + * @param processor processes log-file events + * @param protocolFactory protocol factory + * @param inputTransport file transport + */ + TFileProcessor(boost::shared_ptr processor, + boost::shared_ptr protocolFactory, + boost::shared_ptr inputTransport); + + TFileProcessor(boost::shared_ptr processor, + boost::shared_ptr inputProtocolFactory, + boost::shared_ptr outputProtocolFactory, + boost::shared_ptr inputTransport); + + /** + * Constructor + * + * @param processor processes log-file events + * @param protocolFactory protocol factory + * @param inputTransport input file transport + * @param output output transport + */ + TFileProcessor(boost::shared_ptr processor, + boost::shared_ptr protocolFactory, + boost::shared_ptr inputTransport, + boost::shared_ptr outputTransport); + + /** + * processes events from the file + * + * @param numEvents number of events to process (0 for unlimited) + * @param tail tails the file if true + */ + void process(uint32_t numEvents, bool tail); + + /** + * process events until the end of the chunk + * + */ + void processChunk(); + + private: + boost::shared_ptr processor_; + boost::shared_ptr inputProtocolFactory_; + boost::shared_ptr outputProtocolFactory_; + boost::shared_ptr inputTransport_; + boost::shared_ptr outputTransport_; +}; + + +}}} // apache::thrift::transport + +#endif // _THRIFT_TRANSPORT_TFILETRANSPORT_H_ diff --git a/lib/cpp/src/transport/THttpClient.cpp b/lib/cpp/src/transport/THttpClient.cpp new file mode 100644 index 00000000..59f23396 --- /dev/null +++ b/lib/cpp/src/transport/THttpClient.cpp @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include "THttpClient.h" +#include "TSocket.h" + +namespace apache { namespace thrift { namespace transport { + +using namespace std; + +/** + * Http client implementation. + * + */ + +// Yeah, yeah, hacky to put these here, I know. +static const char* CRLF = "\r\n"; +static const int CRLF_LEN = 2; + +THttpClient::THttpClient(boost::shared_ptr transport, string host, string path) : + transport_(transport), + host_(host), + path_(path), + readHeaders_(true), + chunked_(false), + chunkedDone_(false), + chunkSize_(0), + contentLength_(0), + httpBuf_(NULL), + httpPos_(0), + httpBufLen_(0), + httpBufSize_(1024) { + init(); +} + +THttpClient::THttpClient(string host, int port, string path) : + host_(host), + path_(path), + readHeaders_(true), + chunked_(false), + chunkedDone_(false), + chunkSize_(0), + contentLength_(0), + httpBuf_(NULL), + httpPos_(0), + httpBufLen_(0), + httpBufSize_(1024) { + transport_ = boost::shared_ptr(new TSocket(host, port)); + init(); +} + +void THttpClient::init() { + httpBuf_ = (char*)std::malloc(httpBufSize_+1); + if (httpBuf_ == NULL) { + throw TTransportException("Out of memory."); + } + httpBuf_[httpBufLen_] = '\0'; +} + +THttpClient::~THttpClient() { + if (httpBuf_ != NULL) { + std::free(httpBuf_); + } +} + +uint32_t THttpClient::read(uint8_t* buf, uint32_t len) { + if (readBuffer_.available_read() == 0) { + readBuffer_.resetBuffer(); + uint32_t got = readMoreData(); + if (got == 0) { + return 0; + } + } + return readBuffer_.read(buf, len); +} + +void THttpClient::readEnd() { + // Read any pending chunked data (footers etc.) + if (chunked_) { + while (!chunkedDone_) { + readChunked(); + } + } +} + +uint32_t THttpClient::readMoreData() { + // Get more data! + refill(); + + if (readHeaders_) { + readHeaders(); + } + + if (chunked_) { + return readChunked(); + } else { + return readContent(contentLength_); + } +} + +uint32_t THttpClient::readChunked() { + uint32_t length = 0; + + char* line = readLine(); + uint32_t chunkSize = parseChunkSize(line); + if (chunkSize == 0) { + readChunkedFooters(); + } else { + // Read data content + length += readContent(chunkSize); + // Read trailing CRLF after content + readLine(); + } + return length; +} + +void THttpClient::readChunkedFooters() { + // End of data, read footer lines until a blank one appears + while (true) { + char* line = readLine(); + if (strlen(line) == 0) { + chunkedDone_ = true; + break; + } + } +} + +uint32_t THttpClient::parseChunkSize(char* line) { + char* semi = strchr(line, ';'); + if (semi != NULL) { + *semi = '\0'; + } + int size = 0; + sscanf(line, "%x", &size); + return (uint32_t)size; +} + +uint32_t THttpClient::readContent(uint32_t size) { + uint32_t need = size; + while (need > 0) { + uint32_t avail = httpBufLen_ - httpPos_; + if (avail == 0) { + // We have given all the data, reset position to head of the buffer + httpPos_ = 0; + httpBufLen_ = 0; + refill(); + + // Now have available however much we read + avail = httpBufLen_; + } + uint32_t give = avail; + if (need < give) { + give = need; + } + readBuffer_.write((uint8_t*)(httpBuf_+httpPos_), give); + httpPos_ += give; + need -= give; + } + return size; +} + +char* THttpClient::readLine() { + while (true) { + char* eol = NULL; + + eol = strstr(httpBuf_+httpPos_, CRLF); + + // No CRLF yet? + if (eol == NULL) { + // Shift whatever we have now to front and refill + shift(); + refill(); + } else { + // Return pointer to next line + *eol = '\0'; + char* line = httpBuf_+httpPos_; + httpPos_ = (eol-httpBuf_) + CRLF_LEN; + return line; + } + } + +} + +void THttpClient::shift() { + if (httpBufLen_ > httpPos_) { + // Shift down remaining data and read more + uint32_t length = httpBufLen_ - httpPos_; + memmove(httpBuf_, httpBuf_+httpPos_, length); + httpBufLen_ = length; + } else { + httpBufLen_ = 0; + } + httpPos_ = 0; + httpBuf_[httpBufLen_] = '\0'; +} + +void THttpClient::refill() { + uint32_t avail = httpBufSize_ - httpBufLen_; + if (avail <= (httpBufSize_ / 4)) { + httpBufSize_ *= 2; + httpBuf_ = (char*)std::realloc(httpBuf_, httpBufSize_+1); + if (httpBuf_ == NULL) { + throw TTransportException("Out of memory."); + } + } + + // Read more data + uint32_t got = transport_->read((uint8_t*)(httpBuf_+httpBufLen_), httpBufSize_-httpBufLen_); + httpBufLen_ += got; + httpBuf_[httpBufLen_] = '\0'; + + if (got == 0) { + throw TTransportException("Could not refill buffer"); + } +} + +void THttpClient::readHeaders() { + // Initialize headers state variables + contentLength_ = 0; + chunked_ = false; + chunkedDone_ = false; + chunkSize_ = 0; + + // Control state flow + bool statusLine = true; + bool finished = false; + + // Loop until headers are finished + while (true) { + char* line = readLine(); + + if (strlen(line) == 0) { + if (finished) { + readHeaders_ = false; + return; + } else { + // Must have been an HTTP 100, keep going for another status line + statusLine = true; + } + } else { + if (statusLine) { + statusLine = false; + finished = parseStatusLine(line); + } else { + parseHeader(line); + } + } + } +} + +bool THttpClient::parseStatusLine(char* status) { + char* http = status; + + char* code = strchr(http, ' '); + if (code == NULL) { + throw TTransportException(string("Bad Status: ") + status); + } + + *code = '\0'; + while (*(code++) == ' '); + + char* msg = strchr(code, ' '); + if (msg == NULL) { + throw TTransportException(string("Bad Status: ") + status); + } + *msg = '\0'; + + if (strcmp(code, "200") == 0) { + // HTTP 200 = OK, we got the response + return true; + } else if (strcmp(code, "100") == 0) { + // HTTP 100 = continue, just keep reading + return false; + } else { + throw TTransportException(string("Bad Status: ") + status); + } +} + +void THttpClient::parseHeader(char* header) { + char* colon = strchr(header, ':'); + if (colon == NULL) { + return; + } + uint32_t sz = colon - header; + char* value = colon+1; + + if (strncmp(header, "Transfer-Encoding", sz) == 0) { + if (strstr(value, "chunked") != NULL) { + chunked_ = true; + } + } else if (strncmp(header, "Content-Length", sz) == 0) { + chunked_ = false; + contentLength_ = atoi(value); + } +} + +void THttpClient::write(const uint8_t* buf, uint32_t len) { + writeBuffer_.write(buf, len); +} + +void THttpClient::flush() { + // Fetch the contents of the write buffer + uint8_t* buf; + uint32_t len; + writeBuffer_.getBuffer(&buf, &len); + + // Construct the HTTP header + std::ostringstream h; + h << + "POST " << path_ << " HTTP/1.1" << CRLF << + "Host: " << host_ << CRLF << + "Content-Type: application/x-thrift" << CRLF << + "Content-Length: " << len << CRLF << + "Accept: application/x-thrift" << CRLF << + "User-Agent: C++/THttpClient" << CRLF << + CRLF; + string header = h.str(); + + // Write the header, then the data, then flush + transport_->write((const uint8_t*)header.c_str(), header.size()); + transport_->write(buf, len); + transport_->flush(); + + // Reset the buffer and header variables + writeBuffer_.resetBuffer(); + readHeaders_ = true; +} + +}}} // apache::thrift::transport diff --git a/lib/cpp/src/transport/THttpClient.h b/lib/cpp/src/transport/THttpClient.h new file mode 100644 index 00000000..f4be4c1a --- /dev/null +++ b/lib/cpp/src/transport/THttpClient.h @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_THTTPCLIENT_H_ +#define _THRIFT_TRANSPORT_THTTPCLIENT_H_ 1 + +#include + +namespace apache { namespace thrift { namespace transport { + +/** + * HTTP client implementation of the thrift transport. This was irritating + * to write, but the alternatives in C++ land are daunting. Linking CURL + * requires 23 dynamic libraries last time I checked (WTF?!?). All we have + * here is a VERY basic HTTP/1.1 client which supports HTTP 100 Continue, + * chunked transfer encoding, keepalive, etc. Tested against Apache. + * + */ +class THttpClient : public TTransport { + public: + THttpClient(boost::shared_ptr transport, std::string host, std::string path=""); + + THttpClient(std::string host, int port, std::string path=""); + + virtual ~THttpClient(); + + void open() { + transport_->open(); + } + + bool isOpen() { + return transport_->isOpen(); + } + + bool peek() { + return transport_->peek(); + } + + void close() { + transport_->close(); + } + + uint32_t read(uint8_t* buf, uint32_t len); + + void readEnd(); + + void write(const uint8_t* buf, uint32_t len); + + void flush(); + + private: + void init(); + + protected: + + boost::shared_ptr transport_; + + TMemoryBuffer writeBuffer_; + TMemoryBuffer readBuffer_; + + std::string host_; + std::string path_; + + bool readHeaders_; + bool chunked_; + bool chunkedDone_; + uint32_t chunkSize_; + uint32_t contentLength_; + + char* httpBuf_; + uint32_t httpPos_; + uint32_t httpBufLen_; + uint32_t httpBufSize_; + + uint32_t readMoreData(); + char* readLine(); + + void readHeaders(); + void parseHeader(char* header); + bool parseStatusLine(char* status); + + uint32_t readChunked(); + void readChunkedFooters(); + uint32_t parseChunkSize(char* line); + + uint32_t readContent(uint32_t size); + + void refill(); + void shift(); + +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_THTTPCLIENT_H_ diff --git a/lib/cpp/src/transport/TServerSocket.cpp b/lib/cpp/src/transport/TServerSocket.cpp new file mode 100644 index 00000000..9b47aa53 --- /dev/null +++ b/lib/cpp/src/transport/TServerSocket.cpp @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "TSocket.h" +#include "TServerSocket.h" +#include + +namespace apache { namespace thrift { namespace transport { + +using namespace std; +using boost::shared_ptr; + +TServerSocket::TServerSocket(int port) : + port_(port), + serverSocket_(-1), + acceptBacklog_(1024), + sendTimeout_(0), + recvTimeout_(0), + retryLimit_(0), + retryDelay_(0), + tcpSendBuffer_(0), + tcpRecvBuffer_(0), + intSock1_(-1), + intSock2_(-1) {} + +TServerSocket::TServerSocket(int port, int sendTimeout, int recvTimeout) : + port_(port), + serverSocket_(-1), + acceptBacklog_(1024), + sendTimeout_(sendTimeout), + recvTimeout_(recvTimeout), + retryLimit_(0), + retryDelay_(0), + tcpSendBuffer_(0), + tcpRecvBuffer_(0), + intSock1_(-1), + intSock2_(-1) {} + +TServerSocket::~TServerSocket() { + close(); +} + +void TServerSocket::setSendTimeout(int sendTimeout) { + sendTimeout_ = sendTimeout; +} + +void TServerSocket::setRecvTimeout(int recvTimeout) { + recvTimeout_ = recvTimeout; +} + +void TServerSocket::setRetryLimit(int retryLimit) { + retryLimit_ = retryLimit; +} + +void TServerSocket::setRetryDelay(int retryDelay) { + retryDelay_ = retryDelay; +} + +void TServerSocket::setTcpSendBuffer(int tcpSendBuffer) { + tcpSendBuffer_ = tcpSendBuffer; +} + +void TServerSocket::setTcpRecvBuffer(int tcpRecvBuffer) { + tcpRecvBuffer_ = tcpRecvBuffer; +} + +void TServerSocket::listen() { + int sv[2]; + if (-1 == socketpair(AF_LOCAL, SOCK_STREAM, 0, sv)) { + GlobalOutput.perror("TServerSocket::listen() socketpair() ", errno); + intSock1_ = -1; + intSock2_ = -1; + } else { + intSock1_ = sv[1]; + intSock2_ = sv[0]; + } + + struct addrinfo hints, *res, *res0; + int error; + char port[sizeof("65536") + 1]; + std::memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG; + sprintf(port, "%d", port_); + + // Wildcard address + error = getaddrinfo(NULL, port, &hints, &res0); + if (error) { + GlobalOutput.printf("getaddrinfo %d: %s", error, gai_strerror(error)); + close(); + throw TTransportException(TTransportException::NOT_OPEN, "Could not resolve host for server socket."); + } + + // Pick the ipv6 address first since ipv4 addresses can be mapped + // into ipv6 space. + for (res = res0; res; res = res->ai_next) { + if (res->ai_family == AF_INET6 || res->ai_next == NULL) + break; + } + + serverSocket_ = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if (serverSocket_ == -1) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::listen() socket() ", errno_copy); + close(); + throw TTransportException(TTransportException::NOT_OPEN, "Could not create server socket.", errno_copy); + } + + // Set reusaddress to prevent 2MSL delay on accept + int one = 1; + if (-1 == setsockopt(serverSocket_, SOL_SOCKET, SO_REUSEADDR, + &one, sizeof(one))) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::listen() setsockopt() SO_REUSEADDR ", errno_copy); + close(); + throw TTransportException(TTransportException::NOT_OPEN, "Could not set SO_REUSEADDR", errno_copy); + } + + // Set TCP buffer sizes + if (tcpSendBuffer_ > 0) { + if (-1 == setsockopt(serverSocket_, SOL_SOCKET, SO_SNDBUF, + &tcpSendBuffer_, sizeof(tcpSendBuffer_))) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::listen() setsockopt() SO_SNDBUF ", errno_copy); + close(); + throw TTransportException(TTransportException::NOT_OPEN, "Could not set SO_SNDBUF", errno_copy); + } + } + + if (tcpRecvBuffer_ > 0) { + if (-1 == setsockopt(serverSocket_, SOL_SOCKET, SO_RCVBUF, + &tcpRecvBuffer_, sizeof(tcpRecvBuffer_))) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::listen() setsockopt() SO_RCVBUF ", errno_copy); + close(); + throw TTransportException(TTransportException::NOT_OPEN, "Could not set SO_RCVBUF", errno_copy); + } + } + + // Defer accept + #ifdef TCP_DEFER_ACCEPT + if (-1 == setsockopt(serverSocket_, SOL_SOCKET, TCP_DEFER_ACCEPT, + &one, sizeof(one))) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::listen() setsockopt() TCP_DEFER_ACCEPT ", errno_copy); + close(); + throw TTransportException(TTransportException::NOT_OPEN, "Could not set TCP_DEFER_ACCEPT", errno_copy); + } + #endif // #ifdef TCP_DEFER_ACCEPT + + #ifdef IPV6_V6ONLY + int zero = 0; + if (-1 == setsockopt(serverSocket_, IPPROTO_IPV6, IPV6_V6ONLY, + &zero, sizeof(zero))) { + GlobalOutput.perror("TServerSocket::listen() IPV6_V6ONLY ", errno); + } + #endif // #ifdef IPV6_V6ONLY + + // Turn linger off, don't want to block on calls to close + struct linger ling = {0, 0}; + if (-1 == setsockopt(serverSocket_, SOL_SOCKET, SO_LINGER, + &ling, sizeof(ling))) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::listen() setsockopt() SO_LINGER ", errno_copy); + close(); + throw TTransportException(TTransportException::NOT_OPEN, "Could not set SO_LINGER", errno_copy); + } + + // TCP Nodelay, speed over bandwidth + if (-1 == setsockopt(serverSocket_, IPPROTO_TCP, TCP_NODELAY, + &one, sizeof(one))) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::listen() setsockopt() TCP_NODELAY ", errno_copy); + close(); + throw TTransportException(TTransportException::NOT_OPEN, "Could not set TCP_NODELAY", errno_copy); + } + + // Set NONBLOCK on the accept socket + int flags = fcntl(serverSocket_, F_GETFL, 0); + if (flags == -1) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::listen() fcntl() F_GETFL ", errno_copy); + throw TTransportException(TTransportException::NOT_OPEN, "fcntl() failed", errno_copy); + } + + if (-1 == fcntl(serverSocket_, F_SETFL, flags | O_NONBLOCK)) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::listen() fcntl() O_NONBLOCK ", errno_copy); + throw TTransportException(TTransportException::NOT_OPEN, "fcntl() failed", errno_copy); + } + + // prepare the port information + // we may want to try to bind more than once, since SO_REUSEADDR doesn't + // always seem to work. The client can configure the retry variables. + int retries = 0; + do { + if (0 == bind(serverSocket_, res->ai_addr, res->ai_addrlen)) { + break; + } + + // use short circuit evaluation here to only sleep if we need to + } while ((retries++ < retryLimit_) && (sleep(retryDelay_) == 0)); + + // free addrinfo + freeaddrinfo(res0); + + // throw an error if we failed to bind properly + if (retries > retryLimit_) { + char errbuf[1024]; + sprintf(errbuf, "TServerSocket::listen() BIND %d", port_); + GlobalOutput(errbuf); + close(); + throw TTransportException(TTransportException::NOT_OPEN, "Could not bind"); + } + + // Call listen + if (-1 == ::listen(serverSocket_, acceptBacklog_)) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::listen() listen() ", errno_copy); + close(); + throw TTransportException(TTransportException::NOT_OPEN, "Could not listen", errno_copy); + } + + // The socket is now listening! +} + +shared_ptr TServerSocket::acceptImpl() { + if (serverSocket_ < 0) { + throw TTransportException(TTransportException::NOT_OPEN, "TServerSocket not listening"); + } + + struct pollfd fds[2]; + + int maxEintrs = 5; + int numEintrs = 0; + + while (true) { + std::memset(fds, 0 , sizeof(fds)); + fds[0].fd = serverSocket_; + fds[0].events = POLLIN; + if (intSock2_ >= 0) { + fds[1].fd = intSock2_; + fds[1].events = POLLIN; + } + int ret = poll(fds, 2, -1); + + if (ret < 0) { + // error cases + if (errno == EINTR && (numEintrs++ < maxEintrs)) { + // EINTR needs to be handled manually and we can tolerate + // a certain number + continue; + } + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::acceptImpl() poll() ", errno_copy); + throw TTransportException(TTransportException::UNKNOWN, "Unknown", errno_copy); + } else if (ret > 0) { + // Check for an interrupt signal + if (intSock2_ >= 0 && (fds[1].revents & POLLIN)) { + int8_t buf; + if (-1 == recv(intSock2_, &buf, sizeof(int8_t), 0)) { + GlobalOutput.perror("TServerSocket::acceptImpl() recv() interrupt ", errno); + } + throw TTransportException(TTransportException::INTERRUPTED); + } + + // Check for the actual server socket being ready + if (fds[0].revents & POLLIN) { + break; + } + } else { + GlobalOutput("TServerSocket::acceptImpl() poll 0"); + throw TTransportException(TTransportException::UNKNOWN); + } + } + + struct sockaddr_storage clientAddress; + int size = sizeof(clientAddress); + int clientSocket = ::accept(serverSocket_, + (struct sockaddr *) &clientAddress, + (socklen_t *) &size); + + if (clientSocket < 0) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::acceptImpl() ::accept() ", errno_copy); + throw TTransportException(TTransportException::UNKNOWN, "accept()", errno_copy); + } + + // Make sure client socket is blocking + int flags = fcntl(clientSocket, F_GETFL, 0); + if (flags == -1) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::acceptImpl() fcntl() F_GETFL ", errno_copy); + throw TTransportException(TTransportException::UNKNOWN, "fcntl(F_GETFL)", errno_copy); + } + + if (-1 == fcntl(clientSocket, F_SETFL, flags & ~O_NONBLOCK)) { + int errno_copy = errno; + GlobalOutput.perror("TServerSocket::acceptImpl() fcntl() F_SETFL ~O_NONBLOCK ", errno_copy); + throw TTransportException(TTransportException::UNKNOWN, "fcntl(F_SETFL)", errno_copy); + } + + shared_ptr client(new TSocket(clientSocket)); + if (sendTimeout_ > 0) { + client->setSendTimeout(sendTimeout_); + } + if (recvTimeout_ > 0) { + client->setRecvTimeout(recvTimeout_); + } + + return client; +} + +void TServerSocket::interrupt() { + if (intSock1_ >= 0) { + int8_t byte = 0; + if (-1 == send(intSock1_, &byte, sizeof(int8_t), 0)) { + GlobalOutput.perror("TServerSocket::interrupt() send() ", errno); + } + } +} + +void TServerSocket::close() { + if (serverSocket_ >= 0) { + shutdown(serverSocket_, SHUT_RDWR); + ::close(serverSocket_); + } + if (intSock1_ >= 0) { + ::close(intSock1_); + } + if (intSock2_ >= 0) { + ::close(intSock2_); + } + serverSocket_ = -1; + intSock1_ = -1; + intSock2_ = -1; +} + +}}} // apache::thrift::transport diff --git a/lib/cpp/src/transport/TServerSocket.h b/lib/cpp/src/transport/TServerSocket.h new file mode 100644 index 00000000..a6be0173 --- /dev/null +++ b/lib/cpp/src/transport/TServerSocket.h @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TSERVERSOCKET_H_ +#define _THRIFT_TRANSPORT_TSERVERSOCKET_H_ 1 + +#include "TServerTransport.h" +#include + +namespace apache { namespace thrift { namespace transport { + +class TSocket; + +/** + * Server socket implementation of TServerTransport. Wrapper around a unix + * socket listen and accept calls. + * + */ +class TServerSocket : public TServerTransport { + public: + TServerSocket(int port); + TServerSocket(int port, int sendTimeout, int recvTimeout); + + ~TServerSocket(); + + void setSendTimeout(int sendTimeout); + void setRecvTimeout(int recvTimeout); + + void setRetryLimit(int retryLimit); + void setRetryDelay(int retryDelay); + + void setTcpSendBuffer(int tcpSendBuffer); + void setTcpRecvBuffer(int tcpRecvBuffer); + + void listen(); + void close(); + + void interrupt(); + + protected: + boost::shared_ptr acceptImpl(); + + private: + int port_; + int serverSocket_; + int acceptBacklog_; + int sendTimeout_; + int recvTimeout_; + int retryLimit_; + int retryDelay_; + int tcpSendBuffer_; + int tcpRecvBuffer_; + + int intSock1_; + int intSock2_; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TSERVERSOCKET_H_ diff --git a/lib/cpp/src/transport/TServerTransport.h b/lib/cpp/src/transport/TServerTransport.h new file mode 100644 index 00000000..40bbc6c7 --- /dev/null +++ b/lib/cpp/src/transport/TServerTransport.h @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TSERVERTRANSPORT_H_ +#define _THRIFT_TRANSPORT_TSERVERTRANSPORT_H_ 1 + +#include "TTransport.h" +#include "TTransportException.h" +#include + +namespace apache { namespace thrift { namespace transport { + +/** + * Server transport framework. A server needs to have some facility for + * creating base transports to read/write from. + * + */ +class TServerTransport { + public: + virtual ~TServerTransport() {} + + /** + * Starts the server transport listening for new connections. Prior to this + * call most transports will not return anything when accept is called. + * + * @throws TTransportException if we were unable to listen + */ + virtual void listen() {} + + /** + * Gets a new dynamically allocated transport object and passes it to the + * caller. Note that it is the explicit duty of the caller to free the + * allocated object. The returned TTransport object must always be in the + * opened state. NULL should never be returned, instead an Exception should + * always be thrown. + * + * @return A new TTransport object + * @throws TTransportException if there is an error + */ + boost::shared_ptr accept() { + boost::shared_ptr result = acceptImpl(); + if (result == NULL) { + throw TTransportException("accept() may not return NULL"); + } + return result; + } + + /** + * For "smart" TServerTransport implementations that work in a multi + * threaded context this can be used to break out of an accept() call. + * It is expected that the transport will throw a TTransportException + * with the interrupted error code. + */ + virtual void interrupt() {} + + /** + * Closes this transport such that future calls to accept will do nothing. + */ + virtual void close() = 0; + + protected: + TServerTransport() {} + + /** + * Subclasses should implement this function for accept. + * + * @return A newly allocated TTransport object + * @throw TTransportException If an error occurs + */ + virtual boost::shared_ptr acceptImpl() = 0; + +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TSERVERTRANSPORT_H_ diff --git a/lib/cpp/src/transport/TShortReadTransport.h b/lib/cpp/src/transport/TShortReadTransport.h new file mode 100644 index 00000000..3df8a57c --- /dev/null +++ b/lib/cpp/src/transport/TShortReadTransport.h @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TSHORTREADTRANSPORT_H_ +#define _THRIFT_TRANSPORT_TSHORTREADTRANSPORT_H_ 1 + +#include + +#include + +namespace apache { namespace thrift { namespace transport { namespace test { + +/** + * This class is only meant for testing. It wraps another transport. + * Calls to read are passed through with some probability. Otherwise, + * the read amount is randomly reduced before being passed through. + * + */ +class TShortReadTransport : public TTransport { + public: + TShortReadTransport(boost::shared_ptr transport, double full_prob) + : transport_(transport) + , fullProb_(full_prob) + {} + + bool isOpen() { + return transport_->isOpen(); + } + + bool peek() { + return transport_->peek(); + } + + void open() { + transport_->open(); + } + + void close() { + transport_->close(); + } + + uint32_t read(uint8_t* buf, uint32_t len) { + if (len == 0) { + return 0; + } + + if (rand()/(double)RAND_MAX >= fullProb_) { + len = 1 + rand()%len; + } + return transport_->read(buf, len); + } + + void write(const uint8_t* buf, uint32_t len) { + transport_->write(buf, len); + } + + void flush() { + transport_->flush(); + } + + const uint8_t* borrow(uint8_t* buf, uint32_t* len) { + return transport_->borrow(buf, len); + } + + void consume(uint32_t len) { + return transport_->consume(len); + } + + boost::shared_ptr getUnderlyingTransport() { + return transport_; + } + + protected: + boost::shared_ptr transport_; + double fullProb_; +}; + +}}}} // apache::thrift::transport::test + +#endif // #ifndef _THRIFT_TRANSPORT_TSHORTREADTRANSPORT_H_ diff --git a/lib/cpp/src/transport/TSimpleFileTransport.cpp b/lib/cpp/src/transport/TSimpleFileTransport.cpp new file mode 100644 index 00000000..e58a5743 --- /dev/null +++ b/lib/cpp/src/transport/TSimpleFileTransport.cpp @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "TSimpleFileTransport.h" + +#include +#include +#include + +namespace apache { namespace thrift { namespace transport { + +TSimpleFileTransport:: +TSimpleFileTransport(const std::string& path, bool read, bool write) + : TFDTransport(-1, TFDTransport::CLOSE_ON_DESTROY) { + int flags = 0; + if (read && write) { + flags = O_RDWR; + } else if (read) { + flags = O_RDONLY; + } else if (write) { + flags = O_WRONLY; + } else { + throw TTransportException("Neither READ nor WRITE specified"); + } + if (write) { + flags |= O_CREAT | O_APPEND; + } + int fd = ::open(path.c_str(), + flags, + S_IRUSR | S_IWUSR| S_IRGRP | S_IROTH); + if (fd < 0) { + throw TTransportException("failed to open file for writing: " + path); + } + setFD(fd); + open(); +} + +}}} // apache::thrift::transport diff --git a/lib/cpp/src/transport/TSimpleFileTransport.h b/lib/cpp/src/transport/TSimpleFileTransport.h new file mode 100644 index 00000000..6cc52ea1 --- /dev/null +++ b/lib/cpp/src/transport/TSimpleFileTransport.h @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TSIMPLEFILETRANSPORT_H_ +#define _THRIFT_TRANSPORT_TSIMPLEFILETRANSPORT_H_ 1 + +#include "TFDTransport.h" + +namespace apache { namespace thrift { namespace transport { + +/** + * Dead-simple wrapper around a file. + * + * Writeable files are opened with O_CREAT and O_APPEND + */ +class TSimpleFileTransport : public TFDTransport { + public: + TSimpleFileTransport(const std::string& path, + bool read = true, + bool write = false); +}; + +}}} // apache::thrift::transport + +#endif // _THRIFT_TRANSPORT_TSIMPLEFILETRANSPORT_H_ diff --git a/lib/cpp/src/transport/TSocket.cpp b/lib/cpp/src/transport/TSocket.cpp new file mode 100644 index 00000000..3395dabd --- /dev/null +++ b/lib/cpp/src/transport/TSocket.cpp @@ -0,0 +1,589 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "concurrency/Monitor.h" +#include "TSocket.h" +#include "TTransportException.h" + +namespace apache { namespace thrift { namespace transport { + +using namespace std; + +// Global var to track total socket sys calls +uint32_t g_socket_syscalls = 0; + +/** + * TSocket implementation. + * + */ + +TSocket::TSocket(string host, int port) : + host_(host), + port_(port), + socket_(-1), + connTimeout_(0), + sendTimeout_(0), + recvTimeout_(0), + lingerOn_(1), + lingerVal_(0), + noDelay_(1), + maxRecvRetries_(5) { + recvTimeval_.tv_sec = (int)(recvTimeout_/1000); + recvTimeval_.tv_usec = (int)((recvTimeout_%1000)*1000); +} + +TSocket::TSocket() : + host_(""), + port_(0), + socket_(-1), + connTimeout_(0), + sendTimeout_(0), + recvTimeout_(0), + lingerOn_(1), + lingerVal_(0), + noDelay_(1), + maxRecvRetries_(5) { + recvTimeval_.tv_sec = (int)(recvTimeout_/1000); + recvTimeval_.tv_usec = (int)((recvTimeout_%1000)*1000); +} + +TSocket::TSocket(int socket) : + host_(""), + port_(0), + socket_(socket), + connTimeout_(0), + sendTimeout_(0), + recvTimeout_(0), + lingerOn_(1), + lingerVal_(0), + noDelay_(1), + maxRecvRetries_(5) { + recvTimeval_.tv_sec = (int)(recvTimeout_/1000); + recvTimeval_.tv_usec = (int)((recvTimeout_%1000)*1000); +} + +TSocket::~TSocket() { + close(); +} + +bool TSocket::isOpen() { + return (socket_ >= 0); +} + +bool TSocket::peek() { + if (!isOpen()) { + return false; + } + uint8_t buf; + int r = recv(socket_, &buf, 1, MSG_PEEK); + if (r == -1) { + int errno_copy = errno; + #ifdef __FreeBSD__ + /* shigin: + * freebsd returns -1 and ECONNRESET if socket was closed by + * the other side + */ + if (errno_copy == ECONNRESET) + { + close(); + return false; + } + #endif + GlobalOutput.perror("TSocket::peek() recv() " + getSocketInfo(), errno_copy); + throw TTransportException(TTransportException::UNKNOWN, "recv()", errno_copy); + } + return (r > 0); +} + +void TSocket::openConnection(struct addrinfo *res) { + if (isOpen()) { + throw TTransportException(TTransportException::ALREADY_OPEN); + } + + socket_ = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if (socket_ == -1) { + int errno_copy = errno; + GlobalOutput.perror("TSocket::open() socket() " + getSocketInfo(), errno_copy); + throw TTransportException(TTransportException::NOT_OPEN, "socket()", errno_copy); + } + + // Send timeout + if (sendTimeout_ > 0) { + setSendTimeout(sendTimeout_); + } + + // Recv timeout + if (recvTimeout_ > 0) { + setRecvTimeout(recvTimeout_); + } + + // Linger + setLinger(lingerOn_, lingerVal_); + + // No delay + setNoDelay(noDelay_); + + // Set the socket to be non blocking for connect if a timeout exists + int flags = fcntl(socket_, F_GETFL, 0); + if (connTimeout_ > 0) { + if (-1 == fcntl(socket_, F_SETFL, flags | O_NONBLOCK)) { + int errno_copy = errno; + GlobalOutput.perror("TSocket::open() fcntl() " + getSocketInfo(), errno_copy); + throw TTransportException(TTransportException::NOT_OPEN, "fcntl() failed", errno_copy); + } + } else { + if (-1 == fcntl(socket_, F_SETFL, flags & ~O_NONBLOCK)) { + int errno_copy = errno; + GlobalOutput.perror("TSocket::open() fcntl " + getSocketInfo(), errno_copy); + throw TTransportException(TTransportException::NOT_OPEN, "fcntl() failed", errno_copy); + } + } + + // Connect the socket + int ret = connect(socket_, res->ai_addr, res->ai_addrlen); + + // success case + if (ret == 0) { + goto done; + } + + if (errno != EINPROGRESS) { + int errno_copy = errno; + GlobalOutput.perror("TSocket::open() connect() " + getSocketInfo(), errno_copy); + throw TTransportException(TTransportException::NOT_OPEN, "connect() failed", errno_copy); + } + + + struct pollfd fds[1]; + std::memset(fds, 0 , sizeof(fds)); + fds[0].fd = socket_; + fds[0].events = POLLOUT; + ret = poll(fds, 1, connTimeout_); + + if (ret > 0) { + // Ensure the socket is connected and that there are no errors set + int val; + socklen_t lon; + lon = sizeof(int); + int ret2 = getsockopt(socket_, SOL_SOCKET, SO_ERROR, (void *)&val, &lon); + if (ret2 == -1) { + int errno_copy = errno; + GlobalOutput.perror("TSocket::open() getsockopt() " + getSocketInfo(), errno_copy); + throw TTransportException(TTransportException::NOT_OPEN, "getsockopt()", errno_copy); + } + // no errors on socket, go to town + if (val == 0) { + goto done; + } + GlobalOutput.perror("TSocket::open() error on socket (after poll) " + getSocketInfo(), val); + throw TTransportException(TTransportException::NOT_OPEN, "socket open() error", val); + } else if (ret == 0) { + // socket timed out + string errStr = "TSocket::open() timed out " + getSocketInfo(); + GlobalOutput(errStr.c_str()); + throw TTransportException(TTransportException::NOT_OPEN, "open() timed out"); + } else { + // error on poll() + int errno_copy = errno; + GlobalOutput.perror("TSocket::open() poll() " + getSocketInfo(), errno_copy); + throw TTransportException(TTransportException::NOT_OPEN, "poll() failed", errno_copy); + } + + done: + // Set socket back to normal mode (blocking) + fcntl(socket_, F_SETFL, flags); +} + +void TSocket::open() { + if (isOpen()) { + throw TTransportException(TTransportException::ALREADY_OPEN); + } + + // Validate port number + if (port_ < 0 || port_ > 65536) { + throw TTransportException(TTransportException::NOT_OPEN, "Specified port is invalid"); + } + + struct addrinfo hints, *res, *res0; + res = NULL; + res0 = NULL; + int error; + char port[sizeof("65536")]; + std::memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG; + sprintf(port, "%d", port_); + + error = getaddrinfo(host_.c_str(), port, &hints, &res0); + + if (error) { + string errStr = "TSocket::open() getaddrinfo() " + getSocketInfo() + string(gai_strerror(error)); + GlobalOutput(errStr.c_str()); + close(); + throw TTransportException(TTransportException::NOT_OPEN, "Could not resolve host for client socket."); + } + + // Cycle through all the returned addresses until one + // connects or push the exception up. + for (res = res0; res; res = res->ai_next) { + try { + openConnection(res); + break; + } catch (TTransportException& ttx) { + if (res->ai_next) { + close(); + } else { + close(); + freeaddrinfo(res0); // cleanup on failure + throw; + } + } + } + + // Free address structure memory + freeaddrinfo(res0); +} + +void TSocket::close() { + if (socket_ >= 0) { + shutdown(socket_, SHUT_RDWR); + ::close(socket_); + } + socket_ = -1; +} + +uint32_t TSocket::read(uint8_t* buf, uint32_t len) { + if (socket_ < 0) { + throw TTransportException(TTransportException::NOT_OPEN, "Called read on non-open socket"); + } + + int32_t retries = 0; + + // EAGAIN can be signalled both when a timeout has occurred and when + // the system is out of resources (an awesome undocumented feature). + // The following is an approximation of the time interval under which + // EAGAIN is taken to indicate an out of resources error. + uint32_t eagainThresholdMicros = 0; + if (recvTimeout_) { + // if a readTimeout is specified along with a max number of recv retries, then + // the threshold will ensure that the read timeout is not exceeded even in the + // case of resource errors + eagainThresholdMicros = (recvTimeout_*1000)/ ((maxRecvRetries_>0) ? maxRecvRetries_ : 2); + } + + try_again: + // Read from the socket + struct timeval begin; + gettimeofday(&begin, NULL); + int got = recv(socket_, buf, len, 0); + int errno_copy = errno; //gettimeofday can change errno + struct timeval end; + gettimeofday(&end, NULL); + uint32_t readElapsedMicros = (((end.tv_sec - begin.tv_sec) * 1000 * 1000) + + (((uint64_t)(end.tv_usec - begin.tv_usec)))); + ++g_socket_syscalls; + + // Check for error on read + if (got < 0) { + if (errno_copy == EAGAIN) { + // check if this is the lack of resources or timeout case + if (!eagainThresholdMicros || (readElapsedMicros < eagainThresholdMicros)) { + if (retries++ < maxRecvRetries_) { + usleep(50); + goto try_again; + } else { + throw TTransportException(TTransportException::TIMED_OUT, + "EAGAIN (unavailable resources)"); + } + } else { + // infer that timeout has been hit + throw TTransportException(TTransportException::TIMED_OUT, + "EAGAIN (timed out)"); + } + } + + // If interrupted, try again + if (errno_copy == EINTR && retries++ < maxRecvRetries_) { + goto try_again; + } + + // Now it's not a try again case, but a real probblez + GlobalOutput.perror("TSocket::read() recv() " + getSocketInfo(), errno_copy); + + // If we disconnect with no linger time + if (errno_copy == ECONNRESET) { + #ifdef __FreeBSD__ + /* shigin: freebsd doesn't follow POSIX semantic of recv and fails with + * ECONNRESET if peer performed shutdown + */ + close(); + return 0; + #else + throw TTransportException(TTransportException::NOT_OPEN, "ECONNRESET"); + #endif + } + + // This ish isn't open + if (errno_copy == ENOTCONN) { + throw TTransportException(TTransportException::NOT_OPEN, "ENOTCONN"); + } + + // Timed out! + if (errno_copy == ETIMEDOUT) { + throw TTransportException(TTransportException::TIMED_OUT, "ETIMEDOUT"); + } + + // Some other error, whatevz + throw TTransportException(TTransportException::UNKNOWN, "Unknown", errno_copy); + } + + // The remote host has closed the socket + if (got == 0) { + close(); + return 0; + } + + // Pack data into string + return got; +} + +void TSocket::write(const uint8_t* buf, uint32_t len) { + if (socket_ < 0) { + throw TTransportException(TTransportException::NOT_OPEN, "Called write on non-open socket"); + } + + uint32_t sent = 0; + + while (sent < len) { + + int flags = 0; + #ifdef MSG_NOSIGNAL + // Note the use of MSG_NOSIGNAL to suppress SIGPIPE errors, instead we + // check for the EPIPE return condition and close the socket in that case + flags |= MSG_NOSIGNAL; + #endif // ifdef MSG_NOSIGNAL + + int b = send(socket_, buf + sent, len - sent, flags); + ++g_socket_syscalls; + + // Fail on a send error + if (b < 0) { + int errno_copy = errno; + GlobalOutput.perror("TSocket::write() send() " + getSocketInfo(), errno_copy); + + if (errno == EPIPE || errno == ECONNRESET || errno == ENOTCONN) { + close(); + throw TTransportException(TTransportException::NOT_OPEN, "write() send()", errno_copy); + } + + throw TTransportException(TTransportException::UNKNOWN, "write() send()", errno_copy); + } + + // Fail on blocked send + if (b == 0) { + throw TTransportException(TTransportException::NOT_OPEN, "Socket send returned 0."); + } + sent += b; + } +} + +std::string TSocket::getHost() { + return host_; +} + +int TSocket::getPort() { + return port_; +} + +void TSocket::setHost(string host) { + host_ = host; +} + +void TSocket::setPort(int port) { + port_ = port; +} + +void TSocket::setLinger(bool on, int linger) { + lingerOn_ = on; + lingerVal_ = linger; + if (socket_ < 0) { + return; + } + + struct linger l = {(lingerOn_ ? 1 : 0), lingerVal_}; + int ret = setsockopt(socket_, SOL_SOCKET, SO_LINGER, &l, sizeof(l)); + if (ret == -1) { + int errno_copy = errno; // Copy errno because we're allocating memory. + GlobalOutput.perror("TSocket::setLinger() setsockopt() " + getSocketInfo(), errno_copy); + } +} + +void TSocket::setNoDelay(bool noDelay) { + noDelay_ = noDelay; + if (socket_ < 0) { + return; + } + + // Set socket to NODELAY + int v = noDelay_ ? 1 : 0; + int ret = setsockopt(socket_, IPPROTO_TCP, TCP_NODELAY, &v, sizeof(v)); + if (ret == -1) { + int errno_copy = errno; // Copy errno because we're allocating memory. + GlobalOutput.perror("TSocket::setNoDelay() setsockopt() " + getSocketInfo(), errno_copy); + } +} + +void TSocket::setConnTimeout(int ms) { + connTimeout_ = ms; +} + +void TSocket::setRecvTimeout(int ms) { + if (ms < 0) { + char errBuf[512]; + sprintf(errBuf, "TSocket::setRecvTimeout with negative input: %d\n", ms); + GlobalOutput(errBuf); + return; + } + recvTimeout_ = ms; + + if (socket_ < 0) { + return; + } + + recvTimeval_.tv_sec = (int)(recvTimeout_/1000); + recvTimeval_.tv_usec = (int)((recvTimeout_%1000)*1000); + + // Copy because poll may modify + struct timeval r = recvTimeval_; + int ret = setsockopt(socket_, SOL_SOCKET, SO_RCVTIMEO, &r, sizeof(r)); + if (ret == -1) { + int errno_copy = errno; // Copy errno because we're allocating memory. + GlobalOutput.perror("TSocket::setRecvTimeout() setsockopt() " + getSocketInfo(), errno_copy); + } +} + +void TSocket::setSendTimeout(int ms) { + if (ms < 0) { + char errBuf[512]; + sprintf(errBuf, "TSocket::setSendTimeout with negative input: %d\n", ms); + GlobalOutput(errBuf); + return; + } + sendTimeout_ = ms; + + if (socket_ < 0) { + return; + } + + struct timeval s = {(int)(sendTimeout_/1000), + (int)((sendTimeout_%1000)*1000)}; + int ret = setsockopt(socket_, SOL_SOCKET, SO_SNDTIMEO, &s, sizeof(s)); + if (ret == -1) { + int errno_copy = errno; // Copy errno because we're allocating memory. + GlobalOutput.perror("TSocket::setSendTimeout() setsockopt() " + getSocketInfo(), errno_copy); + } +} + +void TSocket::setMaxRecvRetries(int maxRecvRetries) { + maxRecvRetries_ = maxRecvRetries; +} + +string TSocket::getSocketInfo() { + std::ostringstream oss; + oss << ""; + return oss.str(); +} + +std::string TSocket::getPeerHost() { + if (peerHost_.empty()) { + struct sockaddr_storage addr; + socklen_t addrLen = sizeof(addr); + + if (socket_ < 0) { + return host_; + } + + int rv = getpeername(socket_, (sockaddr*) &addr, &addrLen); + + if (rv != 0) { + return peerHost_; + } + + char clienthost[NI_MAXHOST]; + char clientservice[NI_MAXSERV]; + + getnameinfo((sockaddr*) &addr, addrLen, + clienthost, sizeof(clienthost), + clientservice, sizeof(clientservice), 0); + + peerHost_ = clienthost; + } + return peerHost_; +} + +std::string TSocket::getPeerAddress() { + if (peerAddress_.empty()) { + struct sockaddr_storage addr; + socklen_t addrLen = sizeof(addr); + + if (socket_ < 0) { + return peerAddress_; + } + + int rv = getpeername(socket_, (sockaddr*) &addr, &addrLen); + + if (rv != 0) { + return peerAddress_; + } + + char clienthost[NI_MAXHOST]; + char clientservice[NI_MAXSERV]; + + getnameinfo((sockaddr*) &addr, addrLen, + clienthost, sizeof(clienthost), + clientservice, sizeof(clientservice), + NI_NUMERICHOST|NI_NUMERICSERV); + + peerAddress_ = clienthost; + peerPort_ = std::atoi(clientservice); + } + return peerAddress_; +} + +int TSocket::getPeerPort() { + getPeerAddress(); + return peerPort_; +} + +}}} // apache::thrift::transport diff --git a/lib/cpp/src/transport/TSocket.h b/lib/cpp/src/transport/TSocket.h new file mode 100644 index 00000000..b0f445aa --- /dev/null +++ b/lib/cpp/src/transport/TSocket.h @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TSOCKET_H_ +#define _THRIFT_TRANSPORT_TSOCKET_H_ 1 + +#include +#include + +#include "TTransport.h" +#include "TServerSocket.h" + +namespace apache { namespace thrift { namespace transport { + +/** + * TCP Socket implementation of the TTransport interface. + * + */ +class TSocket : public TTransport { + /** + * We allow the TServerSocket acceptImpl() method to access the private + * members of a socket so that it can access the TSocket(int socket) + * constructor which creates a socket object from the raw UNIX socket + * handle. + */ + friend class TServerSocket; + + public: + /** + * Constructs a new socket. Note that this does NOT actually connect the + * socket. + * + */ + TSocket(); + + /** + * Constructs a new socket. Note that this does NOT actually connect the + * socket. + * + * @param host An IP address or hostname to connect to + * @param port The port to connect on + */ + TSocket(std::string host, int port); + + /** + * Destroyes the socket object, closing it if necessary. + */ + virtual ~TSocket(); + + /** + * Whether the socket is alive. + * + * @return Is the socket alive? + */ + bool isOpen(); + + /** + * Calls select on the socket to see if there is more data available. + */ + bool peek(); + + /** + * Creates and opens the UNIX socket. + * + * @throws TTransportException If the socket could not connect + */ + virtual void open(); + + /** + * Shuts down communications on the socket. + */ + void close(); + + /** + * Reads from the underlying socket. + */ + uint32_t read(uint8_t* buf, uint32_t len); + + /** + * Writes to the underlying socket. + */ + void write(const uint8_t* buf, uint32_t len); + + /** + * Get the host that the socket is connected to + * + * @return string host identifier + */ + std::string getHost(); + + /** + * Get the port that the socket is connected to + * + * @return int port number + */ + int getPort(); + + /** + * Set the host that socket will connect to + * + * @param host host identifier + */ + void setHost(std::string host); + + /** + * Set the port that socket will connect to + * + * @param port port number + */ + void setPort(int port); + + /** + * Controls whether the linger option is set on the socket. + * + * @param on Whether SO_LINGER is on + * @param linger If linger is active, the number of seconds to linger for + */ + void setLinger(bool on, int linger); + + /** + * Whether to enable/disable Nagle's algorithm. + * + * @param noDelay Whether or not to disable the algorithm. + * @return + */ + void setNoDelay(bool noDelay); + + /** + * Set the connect timeout + */ + void setConnTimeout(int ms); + + /** + * Set the receive timeout + */ + void setRecvTimeout(int ms); + + /** + * Set the send timeout + */ + void setSendTimeout(int ms); + + /** + * Set the max number of recv retries in case of an EAGAIN + * error + */ + void setMaxRecvRetries(int maxRecvRetries); + + /** + * Get socket information formated as a string + */ + std::string getSocketInfo(); + + /** + * Returns the DNS name of the host to which the socket is connected + */ + std::string getPeerHost(); + + /** + * Returns the address of the host to which the socket is connected + */ + std::string getPeerAddress(); + + /** + * Returns the port of the host to which the socket is connected + **/ + int getPeerPort(); + + + protected: + /** + * Constructor to create socket from raw UNIX handle. Never called directly + * but used by the TServerSocket class. + */ + TSocket(int socket); + + /** connect, called by open */ + void openConnection(struct addrinfo *res); + + /** Host to connect to */ + std::string host_; + + /** Peer hostname */ + std::string peerHost_; + + /** Peer address */ + std::string peerAddress_; + + /** Peer port */ + int peerPort_; + + /** Port number to connect on */ + int port_; + + /** Underlying UNIX socket handle */ + int socket_; + + /** Connect timeout in ms */ + int connTimeout_; + + /** Send timeout in ms */ + int sendTimeout_; + + /** Recv timeout in ms */ + int recvTimeout_; + + /** Linger on */ + bool lingerOn_; + + /** Linger val */ + int lingerVal_; + + /** Nodelay */ + bool noDelay_; + + /** Recv EGAIN retries */ + int maxRecvRetries_; + + /** Recv timeout timeval */ + struct timeval recvTimeval_; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TSOCKET_H_ + diff --git a/lib/cpp/src/transport/TSocketPool.cpp b/lib/cpp/src/transport/TSocketPool.cpp new file mode 100644 index 00000000..1150282b --- /dev/null +++ b/lib/cpp/src/transport/TSocketPool.cpp @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include "TSocketPool.h" + +namespace apache { namespace thrift { namespace transport { + +using namespace std; + +using boost::shared_ptr; + +/** + * TSocketPoolServer implementation + * + */ +TSocketPoolServer::TSocketPoolServer() + : host_(""), + port_(0), + socket_(-1), + lastFailTime_(0), + consecutiveFailures_(0) {} + +/** + * Constructor for TSocketPool server + */ +TSocketPoolServer::TSocketPoolServer(const string &host, int port) + : host_(host), + port_(port), + socket_(-1), + lastFailTime_(0), + consecutiveFailures_(0) {} + +/** + * TSocketPool implementation. + * + */ + +TSocketPool::TSocketPool() : TSocket(), + numRetries_(1), + retryInterval_(60), + maxConsecutiveFailures_(1), + randomize_(true), + alwaysTryLast_(true) { +} + +TSocketPool::TSocketPool(const vector &hosts, + const vector &ports) : TSocket(), + numRetries_(1), + retryInterval_(60), + maxConsecutiveFailures_(1), + randomize_(true), + alwaysTryLast_(true) +{ + if (hosts.size() != ports.size()) { + GlobalOutput("TSocketPool::TSocketPool: hosts.size != ports.size"); + throw TTransportException(TTransportException::BAD_ARGS); + } + + for (unsigned int i = 0; i < hosts.size(); ++i) { + addServer(hosts[i], ports[i]); + } +} + +TSocketPool::TSocketPool(const vector >& servers) : TSocket(), + numRetries_(1), + retryInterval_(60), + maxConsecutiveFailures_(1), + randomize_(true), + alwaysTryLast_(true) +{ + for (unsigned i = 0; i < servers.size(); ++i) { + addServer(servers[i].first, servers[i].second); + } +} + +TSocketPool::TSocketPool(const vector< shared_ptr >& servers) : TSocket(), + servers_(servers), + numRetries_(1), + retryInterval_(60), + maxConsecutiveFailures_(1), + randomize_(true), + alwaysTryLast_(true) +{ +} + +TSocketPool::TSocketPool(const string& host, int port) : TSocket(), + numRetries_(1), + retryInterval_(60), + maxConsecutiveFailures_(1), + randomize_(true), + alwaysTryLast_(true) +{ + addServer(host, port); +} + +TSocketPool::~TSocketPool() { + vector< shared_ptr >::const_iterator iter = servers_.begin(); + vector< shared_ptr >::const_iterator iterEnd = servers_.end(); + for (; iter != iterEnd; ++iter) { + setCurrentServer(*iter); + TSocketPool::close(); + } +} + +void TSocketPool::addServer(const string& host, int port) { + servers_.push_back(shared_ptr(new TSocketPoolServer(host, port))); +} + +void TSocketPool::setServers(const vector< shared_ptr >& servers) { + servers_ = servers; +} + +void TSocketPool::getServers(vector< shared_ptr >& servers) { + servers = servers_; +} + +void TSocketPool::setNumRetries(int numRetries) { + numRetries_ = numRetries; +} + +void TSocketPool::setRetryInterval(int retryInterval) { + retryInterval_ = retryInterval; +} + + +void TSocketPool::setMaxConsecutiveFailures(int maxConsecutiveFailures) { + maxConsecutiveFailures_ = maxConsecutiveFailures; +} + +void TSocketPool::setRandomize(bool randomize) { + randomize_ = randomize; +} + +void TSocketPool::setAlwaysTryLast(bool alwaysTryLast) { + alwaysTryLast_ = alwaysTryLast; +} + +void TSocketPool::setCurrentServer(const shared_ptr &server) { + currentServer_ = server; + host_ = server->host_; + port_ = server->port_; + socket_ = server->socket_; +} + +/* TODO: without apc we ignore a lot of functionality from the php version */ +void TSocketPool::open() { + if (randomize_) { + random_shuffle(servers_.begin(), servers_.end()); + } + + unsigned int numServers = servers_.size(); + for (unsigned int i = 0; i < numServers; ++i) { + + shared_ptr &server = servers_[i]; + bool retryIntervalPassed = (server->lastFailTime_ == 0); + bool isLastServer = alwaysTryLast_ ? (i == (numServers - 1)) : false; + + // Impersonate the server socket + setCurrentServer(server); + + if (isOpen()) { + // already open means we're done + return; + } + + if (server->lastFailTime_ > 0) { + // The server was marked as down, so check if enough time has elapsed to retry + int elapsedTime = time(NULL) - server->lastFailTime_; + if (elapsedTime > retryInterval_) { + retryIntervalPassed = true; + } + } + + if (retryIntervalPassed || isLastServer) { + for (int j = 0; j < numRetries_; ++j) { + try { + TSocket::open(); + + // Copy over the opened socket so that we can keep it persistent + server->socket_ = socket_; + + // reset lastFailTime_ is required + if (server->lastFailTime_) { + server->lastFailTime_ = 0; + } + + // success + return; + } catch (TException e) { + string errStr = "TSocketPool::open failed "+getSocketInfo()+": "+e.what(); + GlobalOutput(errStr.c_str()); + // connection failed + } + } + + ++server->consecutiveFailures_; + if (server->consecutiveFailures_ > maxConsecutiveFailures_) { + // Mark server as down + server->consecutiveFailures_ = 0; + server->lastFailTime_ = time(NULL); + } + } + } + + GlobalOutput("TSocketPool::open: all connections failed"); + throw TTransportException(TTransportException::NOT_OPEN); +} + +void TSocketPool::close() { + if (isOpen()) { + TSocket::close(); + currentServer_->socket_ = -1; + } +} + +}}} // apache::thrift::transport diff --git a/lib/cpp/src/transport/TSocketPool.h b/lib/cpp/src/transport/TSocketPool.h new file mode 100644 index 00000000..8c506695 --- /dev/null +++ b/lib/cpp/src/transport/TSocketPool.h @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TSOCKETPOOL_H_ +#define _THRIFT_TRANSPORT_TSOCKETPOOL_H_ 1 + +#include +#include "TSocket.h" + +namespace apache { namespace thrift { namespace transport { + + /** + * Class to hold server information for TSocketPool + * + */ +class TSocketPoolServer { + + public: + /** + * Default constructor for server info + */ + TSocketPoolServer(); + + /** + * Constructor for TSocketPool server + */ + TSocketPoolServer(const std::string &host, int port); + + // Host name + std::string host_; + + // Port to connect on + int port_; + + // Socket for the server + int socket_; + + // Last time connecting to this server failed + int lastFailTime_; + + // Number of consecutive times connecting to this server failed + int consecutiveFailures_; +}; + +/** + * TCP Socket implementation of the TTransport interface. + * + */ +class TSocketPool : public TSocket { + + public: + + /** + * Socket pool constructor + */ + TSocketPool(); + + /** + * Socket pool constructor + * + * @param hosts list of host names + * @param ports list of port names + */ + TSocketPool(const std::vector &hosts, + const std::vector &ports); + + /** + * Socket pool constructor + * + * @param servers list of pairs of host name and port + */ + TSocketPool(const std::vector >& servers); + + /** + * Socket pool constructor + * + * @param servers list of TSocketPoolServers + */ + TSocketPool(const std::vector< boost::shared_ptr >& servers); + + /** + * Socket pool constructor + * + * @param host single host + * @param port single port + */ + TSocketPool(const std::string& host, int port); + + /** + * Destroyes the socket object, closing it if necessary. + */ + virtual ~TSocketPool(); + + /** + * Add a server to the pool + */ + void addServer(const std::string& host, int port); + + /** + * Set list of servers in this pool + */ + void setServers(const std::vector< boost::shared_ptr >& servers); + + /** + * Get list of servers in this pool + */ + void getServers(std::vector< boost::shared_ptr >& servers); + + /** + * Sets how many times to keep retrying a host in the connect function. + */ + void setNumRetries(int numRetries); + + /** + * Sets how long to wait until retrying a host if it was marked down + */ + void setRetryInterval(int retryInterval); + + /** + * Sets how many times to keep retrying a host before marking it as down. + */ + void setMaxConsecutiveFailures(int maxConsecutiveFailures); + + /** + * Turns randomization in connect order on or off. + */ + void setRandomize(bool randomize); + + /** + * Whether to always try the last server. + */ + void setAlwaysTryLast(bool alwaysTryLast); + + /** + * Creates and opens the UNIX socket. + */ + void open(); + + /* + * Closes the UNIX socket + */ + void close(); + + protected: + + void setCurrentServer(const boost::shared_ptr &server); + + /** List of servers to connect to */ + std::vector< boost::shared_ptr > servers_; + + /** Current server */ + boost::shared_ptr currentServer_; + + /** How many times to retry each host in connect */ + int numRetries_; + + /** Retry interval in seconds, how long to not try a host if it has been + * marked as down. + */ + int retryInterval_; + + /** Max consecutive failures before marking a host down. */ + int maxConsecutiveFailures_; + + /** Try hosts in order? or Randomized? */ + bool randomize_; + + /** Always try last host, even if marked down? */ + bool alwaysTryLast_; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TSOCKETPOOL_H_ + diff --git a/lib/cpp/src/transport/TTransport.h b/lib/cpp/src/transport/TTransport.h new file mode 100644 index 00000000..eb0d5df8 --- /dev/null +++ b/lib/cpp/src/transport/TTransport.h @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TTRANSPORT_H_ +#define _THRIFT_TRANSPORT_TTRANSPORT_H_ 1 + +#include +#include +#include +#include + +namespace apache { namespace thrift { namespace transport { + +/** + * Generic interface for a method of transporting data. A TTransport may be + * capable of either reading or writing, but not necessarily both. + * + */ +class TTransport { + public: + /** + * Virtual deconstructor. + */ + virtual ~TTransport() {} + + /** + * Whether this transport is open. + */ + virtual bool isOpen() { + return false; + } + + /** + * Tests whether there is more data to read or if the remote side is + * still open. By default this is true whenever the transport is open, + * but implementations should add logic to test for this condition where + * possible (i.e. on a socket). + * This is used by a server to check if it should listen for another + * request. + */ + virtual bool peek() { + return isOpen(); + } + + /** + * Opens the transport for communications. + * + * @return bool Whether the transport was successfully opened + * @throws TTransportException if opening failed + */ + virtual void open() { + throw TTransportException(TTransportException::NOT_OPEN, "Cannot open base TTransport."); + } + + /** + * Closes the transport. + */ + virtual void close() { + throw TTransportException(TTransportException::NOT_OPEN, "Cannot close base TTransport."); + } + + /** + * Attempt to read up to the specified number of bytes into the string. + * + * @param buf Reference to the location to write the data + * @param len How many bytes to read + * @return How many bytes were actually read + * @throws TTransportException If an error occurs + */ + virtual uint32_t read(uint8_t* /* buf */, uint32_t /* len */) { + throw TTransportException(TTransportException::NOT_OPEN, "Base TTransport cannot read."); + } + + /** + * Reads the given amount of data in its entirety no matter what. + * + * @param s Reference to location for read data + * @param len How many bytes to read + * @return How many bytes read, which must be equal to size + * @throws TTransportException If insufficient data was read + */ + virtual uint32_t readAll(uint8_t* buf, uint32_t len) { + uint32_t have = 0; + uint32_t get = 0; + + while (have < len) { + get = read(buf+have, len-have); + if (get <= 0) { + throw TTransportException("No more data to read."); + } + have += get; + } + + return have; + } + + /** + * Called when read is completed. + * This can be over-ridden to perform a transport-specific action + * e.g. logging the request to a file + * + */ + virtual void readEnd() { + // default behaviour is to do nothing + return; + } + + /** + * Writes the string in its entirety to the buffer. + * + * @param buf The data to write out + * @throws TTransportException if an error occurs + */ + virtual void write(const uint8_t* /* buf */, uint32_t /* len */) { + throw TTransportException(TTransportException::NOT_OPEN, "Base TTransport cannot write."); + } + + /** + * Called when write is completed. + * This can be over-ridden to perform a transport-specific action + * at the end of a request. + * + */ + virtual void writeEnd() { + // default behaviour is to do nothing + return; + } + + /** + * Flushes any pending data to be written. Typically used with buffered + * transport mechanisms. + * + * @throws TTransportException if an error occurs + */ + virtual void flush() {} + + /** + * Attempts to return a pointer to \c len bytes, possibly copied into \c buf. + * Does not consume the bytes read (i.e.: a later read will return the same + * data). This method is meant to support protocols that need to read + * variable-length fields. They can attempt to borrow the maximum amount of + * data that they will need, then consume (see next method) what they + * actually use. Some transports will not support this method and others + * will fail occasionally, so protocols must be prepared to use read if + * borrow fails. + * + * @oaram buf A buffer where the data can be stored if needed. + * If borrow doesn't return buf, then the contents of + * buf after the call are undefined. + * @param len *len should initially contain the number of bytes to borrow. + * If borrow succeeds, *len will contain the number of bytes + * available in the returned pointer. This will be at least + * what was requested, but may be more if borrow returns + * a pointer to an internal buffer, rather than buf. + * If borrow fails, the contents of *len are undefined. + * @return If the borrow succeeds, return a pointer to the borrowed data. + * This might be equal to \c buf, or it might be a pointer into + * the transport's internal buffers. + * @throws TTransportException if an error occurs + */ + virtual const uint8_t* borrow(uint8_t* /* buf */, uint32_t* /* len */) { + return NULL; + } + + /** + * Remove len bytes from the transport. This should always follow a borrow + * of at least len bytes, and should always succeed. + * TODO(dreiss): Is there any transport that could borrow but fail to + * consume, or that would require a buffer to dump the consumed data? + * + * @param len How many bytes to consume + * @throws TTransportException If an error occurs + */ + virtual void consume(uint32_t /* len */) { + throw TTransportException(TTransportException::NOT_OPEN, "Base TTransport cannot consume."); + } + + protected: + /** + * Simple constructor. + */ + TTransport() {} +}; + +/** + * Generic factory class to make an input and output transport out of a + * source transport. Commonly used inside servers to make input and output + * streams out of raw clients. + * + */ +class TTransportFactory { + public: + TTransportFactory() {} + + virtual ~TTransportFactory() {} + + /** + * Default implementation does nothing, just returns the transport given. + */ + virtual boost::shared_ptr getTransport(boost::shared_ptr trans) { + return trans; + } + +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TTRANSPORT_H_ diff --git a/lib/cpp/src/transport/TTransportException.cpp b/lib/cpp/src/transport/TTransportException.cpp new file mode 100644 index 00000000..f0aaedc2 --- /dev/null +++ b/lib/cpp/src/transport/TTransportException.cpp @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +using std::string; +using boost::lexical_cast; + +namespace apache { namespace thrift { namespace transport { + +}}} // apache::thrift::transport + diff --git a/lib/cpp/src/transport/TTransportException.h b/lib/cpp/src/transport/TTransportException.h new file mode 100644 index 00000000..330785ce --- /dev/null +++ b/lib/cpp/src/transport/TTransportException.h @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_ +#define _THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_ 1 + +#include +#include + +namespace apache { namespace thrift { namespace transport { + +/** + * Class to encapsulate all the possible types of transport errors that may + * occur in various transport systems. This provides a sort of generic + * wrapper around the shitty UNIX E_ error codes that lets a common code + * base of error handling to be used for various types of transports, i.e. + * pipes etc. + * + */ +class TTransportException : public apache::thrift::TException { + public: + /** + * Error codes for the various types of exceptions. + */ + enum TTransportExceptionType + { UNKNOWN = 0 + , NOT_OPEN = 1 + , ALREADY_OPEN = 2 + , TIMED_OUT = 3 + , END_OF_FILE = 4 + , INTERRUPTED = 5 + , BAD_ARGS = 6 + , CORRUPTED_DATA = 7 + , INTERNAL_ERROR = 8 + }; + + TTransportException() : + apache::thrift::TException(), + type_(UNKNOWN) {} + + TTransportException(TTransportExceptionType type) : + apache::thrift::TException(), + type_(type) {} + + TTransportException(const std::string& message) : + apache::thrift::TException(message), + type_(UNKNOWN) {} + + TTransportException(TTransportExceptionType type, const std::string& message) : + apache::thrift::TException(message), + type_(type) {} + + TTransportException(TTransportExceptionType type, + const std::string& message, + int errno_copy) : + apache::thrift::TException(message + ": " + TOutput::strerror_s(errno_copy)), + type_(type) {} + + virtual ~TTransportException() throw() {} + + /** + * Returns an error code that provides information about the type of error + * that has occurred. + * + * @return Error code + */ + TTransportExceptionType getType() const throw() { + return type_; + } + + virtual const char* what() const throw() { + if (message_.empty()) { + switch (type_) { + case UNKNOWN : return "TTransportException: Unknown transport exception"; + case NOT_OPEN : return "TTransportException: Transport not open"; + case ALREADY_OPEN : return "TTransportException: Transport already open"; + case TIMED_OUT : return "TTransportException: Timed out"; + case END_OF_FILE : return "TTransportException: End of file"; + case INTERRUPTED : return "TTransportException: Interrupted"; + case BAD_ARGS : return "TTransportException: Invalid arguments"; + case CORRUPTED_DATA : return "TTransportException: Corrupted Data"; + case INTERNAL_ERROR : return "TTransportException: Internal error"; + default : return "TTransportException: (Invalid exception type)"; + } + } else { + return message_.c_str(); + } + } + + protected: + /** Just like strerror_r but returns a C++ string object. */ + std::string strerror_s(int errno_copy); + + /** Error code */ + TTransportExceptionType type_; + +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TTRANSPORTEXCEPTION_H_ diff --git a/lib/cpp/src/transport/TTransportUtils.cpp b/lib/cpp/src/transport/TTransportUtils.cpp new file mode 100644 index 00000000..a840fa6c --- /dev/null +++ b/lib/cpp/src/transport/TTransportUtils.cpp @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +using std::string; + +namespace apache { namespace thrift { namespace transport { + +uint32_t TPipedTransport::read(uint8_t* buf, uint32_t len) { + uint32_t need = len; + + // We don't have enough data yet + if (rLen_-rPos_ < need) { + // Copy out whatever we have + if (rLen_-rPos_ > 0) { + memcpy(buf, rBuf_+rPos_, rLen_-rPos_); + need -= rLen_-rPos_; + buf += rLen_-rPos_; + rPos_ = rLen_; + } + + // Double the size of the underlying buffer if it is full + if (rLen_ == rBufSize_) { + rBufSize_ *=2; + rBuf_ = (uint8_t *)std::realloc(rBuf_, sizeof(uint8_t) * rBufSize_); + } + + // try to fill up the buffer + rLen_ += srcTrans_->read(rBuf_+rPos_, rBufSize_ - rPos_); + } + + + // Hand over whatever we have + uint32_t give = need; + if (rLen_-rPos_ < give) { + give = rLen_-rPos_; + } + if (give > 0) { + memcpy(buf, rBuf_+rPos_, give); + rPos_ += give; + need -= give; + } + + return (len - need); +} + +void TPipedTransport::write(const uint8_t* buf, uint32_t len) { + if (len == 0) { + return; + } + + // Make the buffer as big as it needs to be + if ((len + wLen_) >= wBufSize_) { + uint32_t newBufSize = wBufSize_*2; + while ((len + wLen_) >= newBufSize) { + newBufSize *= 2; + } + wBuf_ = (uint8_t *)std::realloc(wBuf_, sizeof(uint8_t) * newBufSize); + wBufSize_ = newBufSize; + } + + // Copy into the buffer + memcpy(wBuf_ + wLen_, buf, len); + wLen_ += len; +} + +void TPipedTransport::flush() { + // Write out any data waiting in the write buffer + if (wLen_ > 0) { + srcTrans_->write(wBuf_, wLen_); + wLen_ = 0; + } + + // Flush the underlying transport + srcTrans_->flush(); +} + +TPipedFileReaderTransport::TPipedFileReaderTransport(boost::shared_ptr srcTrans, boost::shared_ptr dstTrans) + : TPipedTransport(srcTrans, dstTrans), + srcTrans_(srcTrans) { +} + +TPipedFileReaderTransport::~TPipedFileReaderTransport() { +} + +bool TPipedFileReaderTransport::isOpen() { + return TPipedTransport::isOpen(); +} + +bool TPipedFileReaderTransport::peek() { + return TPipedTransport::peek(); +} + +void TPipedFileReaderTransport::open() { + TPipedTransport::open(); +} + +void TPipedFileReaderTransport::close() { + TPipedTransport::close(); +} + +uint32_t TPipedFileReaderTransport::read(uint8_t* buf, uint32_t len) { + return TPipedTransport::read(buf, len); +} + +uint32_t TPipedFileReaderTransport::readAll(uint8_t* buf, uint32_t len) { + uint32_t have = 0; + uint32_t get = 0; + + while (have < len) { + get = read(buf+have, len-have); + if (get <= 0) { + throw TEOFException(); + } + have += get; + } + + return have; +} + +void TPipedFileReaderTransport::readEnd() { + TPipedTransport::readEnd(); +} + +void TPipedFileReaderTransport::write(const uint8_t* buf, uint32_t len) { + TPipedTransport::write(buf, len); +} + +void TPipedFileReaderTransport::writeEnd() { + TPipedTransport::writeEnd(); +} + +void TPipedFileReaderTransport::flush() { + TPipedTransport::flush(); +} + +int32_t TPipedFileReaderTransport::getReadTimeout() { + return srcTrans_->getReadTimeout(); +} + +void TPipedFileReaderTransport::setReadTimeout(int32_t readTimeout) { + srcTrans_->setReadTimeout(readTimeout); +} + +uint32_t TPipedFileReaderTransport::getNumChunks() { + return srcTrans_->getNumChunks(); +} + +uint32_t TPipedFileReaderTransport::getCurChunk() { + return srcTrans_->getCurChunk(); +} + +void TPipedFileReaderTransport::seekToChunk(int32_t chunk) { + srcTrans_->seekToChunk(chunk); +} + +void TPipedFileReaderTransport::seekToEnd() { + srcTrans_->seekToEnd(); +} + +}}} // apache::thrift::transport diff --git a/lib/cpp/src/transport/TTransportUtils.h b/lib/cpp/src/transport/TTransportUtils.h new file mode 100644 index 00000000..d65c9167 --- /dev/null +++ b/lib/cpp/src/transport/TTransportUtils.h @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TTRANSPORTUTILS_H_ +#define _THRIFT_TRANSPORT_TTRANSPORTUTILS_H_ 1 + +#include +#include +#include +#include +#include +// Include the buffered transports that used to be defined here. +#include +#include + +namespace apache { namespace thrift { namespace transport { + +/** + * The null transport is a dummy transport that doesn't actually do anything. + * It's sort of an analogy to /dev/null, you can never read anything from it + * and it will let you write anything you want to it, though it won't actually + * go anywhere. + * + */ +class TNullTransport : public TTransport { + public: + TNullTransport() {} + + ~TNullTransport() {} + + bool isOpen() { + return true; + } + + void open() {} + + void write(const uint8_t* /* buf */, uint32_t /* len */) { + return; + } + +}; + + +/** + * TPipedTransport. This transport allows piping of a request from one + * transport to another either when readEnd() or writeEnd(). The typical + * use case for this is to log a request or a reply to disk. + * The underlying buffer expands to a keep a copy of the entire + * request/response. + * + */ +class TPipedTransport : virtual public TTransport { + public: + TPipedTransport(boost::shared_ptr srcTrans, + boost::shared_ptr dstTrans) : + srcTrans_(srcTrans), + dstTrans_(dstTrans), + rBufSize_(512), rPos_(0), rLen_(0), + wBufSize_(512), wLen_(0) { + + // default is to to pipe the request when readEnd() is called + pipeOnRead_ = true; + pipeOnWrite_ = false; + + rBuf_ = (uint8_t*) std::malloc(sizeof(uint8_t) * rBufSize_); + wBuf_ = (uint8_t*) std::malloc(sizeof(uint8_t) * wBufSize_); + } + + TPipedTransport(boost::shared_ptr srcTrans, + boost::shared_ptr dstTrans, + uint32_t sz) : + srcTrans_(srcTrans), + dstTrans_(dstTrans), + rBufSize_(512), rPos_(0), rLen_(0), + wBufSize_(sz), wLen_(0) { + + rBuf_ = (uint8_t*) std::malloc(sizeof(uint8_t) * rBufSize_); + wBuf_ = (uint8_t*) std::malloc(sizeof(uint8_t) * wBufSize_); + } + + ~TPipedTransport() { + std::free(rBuf_); + std::free(wBuf_); + } + + bool isOpen() { + return srcTrans_->isOpen(); + } + + bool peek() { + if (rPos_ >= rLen_) { + // Double the size of the underlying buffer if it is full + if (rLen_ == rBufSize_) { + rBufSize_ *=2; + rBuf_ = (uint8_t *)std::realloc(rBuf_, sizeof(uint8_t) * rBufSize_); + } + + // try to fill up the buffer + rLen_ += srcTrans_->read(rBuf_+rPos_, rBufSize_ - rPos_); + } + return (rLen_ > rPos_); + } + + + void open() { + srcTrans_->open(); + } + + void close() { + srcTrans_->close(); + } + + void setPipeOnRead(bool pipeVal) { + pipeOnRead_ = pipeVal; + } + + void setPipeOnWrite(bool pipeVal) { + pipeOnWrite_ = pipeVal; + } + + uint32_t read(uint8_t* buf, uint32_t len); + + void readEnd() { + + if (pipeOnRead_) { + dstTrans_->write(rBuf_, rPos_); + dstTrans_->flush(); + } + + srcTrans_->readEnd(); + + // If requests are being pipelined, copy down our read-ahead data, + // then reset our state. + int read_ahead = rLen_ - rPos_; + memcpy(rBuf_, rBuf_ + rPos_, read_ahead); + rPos_ = 0; + rLen_ = read_ahead; + } + + void write(const uint8_t* buf, uint32_t len); + + void writeEnd() { + if (pipeOnWrite_) { + dstTrans_->write(wBuf_, wLen_); + dstTrans_->flush(); + } + } + + void flush(); + + boost::shared_ptr getTargetTransport() { + return dstTrans_; + } + + protected: + boost::shared_ptr srcTrans_; + boost::shared_ptr dstTrans_; + + uint8_t* rBuf_; + uint32_t rBufSize_; + uint32_t rPos_; + uint32_t rLen_; + + uint8_t* wBuf_; + uint32_t wBufSize_; + uint32_t wLen_; + + bool pipeOnRead_; + bool pipeOnWrite_; +}; + + +/** + * Wraps a transport into a pipedTransport instance. + * + */ +class TPipedTransportFactory : public TTransportFactory { + public: + TPipedTransportFactory() {} + TPipedTransportFactory(boost::shared_ptr dstTrans) { + initializeTargetTransport(dstTrans); + } + virtual ~TPipedTransportFactory() {} + + /** + * Wraps the base transport into a piped transport. + */ + virtual boost::shared_ptr getTransport(boost::shared_ptr srcTrans) { + return boost::shared_ptr(new TPipedTransport(srcTrans, dstTrans_)); + } + + virtual void initializeTargetTransport(boost::shared_ptr dstTrans) { + if (dstTrans_.get() == NULL) { + dstTrans_ = dstTrans; + } else { + throw TException("Target transport already initialized"); + } + } + + protected: + boost::shared_ptr dstTrans_; +}; + +/** + * TPipedFileTransport. This is just like a TTransport, except that + * it is a templatized class, so that clients who rely on a specific + * TTransport can still access the original transport. + * + */ +class TPipedFileReaderTransport : public TPipedTransport, + public TFileReaderTransport { + public: + TPipedFileReaderTransport(boost::shared_ptr srcTrans, boost::shared_ptr dstTrans); + + ~TPipedFileReaderTransport(); + + // TTransport functions + bool isOpen(); + bool peek(); + void open(); + void close(); + uint32_t read(uint8_t* buf, uint32_t len); + uint32_t readAll(uint8_t* buf, uint32_t len); + void readEnd(); + void write(const uint8_t* buf, uint32_t len); + void writeEnd(); + void flush(); + + // TFileReaderTransport functions + int32_t getReadTimeout(); + void setReadTimeout(int32_t readTimeout); + uint32_t getNumChunks(); + uint32_t getCurChunk(); + void seekToChunk(int32_t chunk); + void seekToEnd(); + + protected: + // shouldn't be used + TPipedFileReaderTransport(); + boost::shared_ptr srcTrans_; +}; + +/** + * Creates a TPipedFileReaderTransport from a filepath and a destination transport + * + */ +class TPipedFileReaderTransportFactory : public TPipedTransportFactory { + public: + TPipedFileReaderTransportFactory() {} + TPipedFileReaderTransportFactory(boost::shared_ptr dstTrans) + : TPipedTransportFactory(dstTrans) + {} + virtual ~TPipedFileReaderTransportFactory() {} + + boost::shared_ptr getTransport(boost::shared_ptr srcTrans) { + boost::shared_ptr pFileReaderTransport = boost::dynamic_pointer_cast(srcTrans); + if (pFileReaderTransport.get() != NULL) { + return getFileReaderTransport(pFileReaderTransport); + } else { + return boost::shared_ptr(); + } + } + + boost::shared_ptr getFileReaderTransport(boost::shared_ptr srcTrans) { + return boost::shared_ptr(new TPipedFileReaderTransport(srcTrans, dstTrans_)); + } +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TTRANSPORTUTILS_H_ diff --git a/lib/cpp/src/transport/TZlibTransport.cpp b/lib/cpp/src/transport/TZlibTransport.cpp new file mode 100644 index 00000000..2f14e906 --- /dev/null +++ b/lib/cpp/src/transport/TZlibTransport.cpp @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +using std::string; + +namespace apache { namespace thrift { namespace transport { + +// Don't call this outside of the constructor. +void TZlibTransport::initZlib() { + int rv; + bool r_init = false; + try { + rstream_ = new z_stream; + wstream_ = new z_stream; + + rstream_->zalloc = Z_NULL; + wstream_->zalloc = Z_NULL; + rstream_->zfree = Z_NULL; + wstream_->zfree = Z_NULL; + rstream_->opaque = Z_NULL; + wstream_->opaque = Z_NULL; + + rstream_->next_in = crbuf_; + wstream_->next_in = uwbuf_; + rstream_->next_out = urbuf_; + wstream_->next_out = cwbuf_; + rstream_->avail_in = 0; + wstream_->avail_in = 0; + rstream_->avail_out = urbuf_size_; + wstream_->avail_out = cwbuf_size_; + + rv = inflateInit(rstream_); + checkZlibRv(rv, rstream_->msg); + + // Have to set this flag so we know whether to de-initialize. + r_init = true; + + rv = deflateInit(wstream_, Z_DEFAULT_COMPRESSION); + checkZlibRv(rv, wstream_->msg); + } + + catch (...) { + if (r_init) { + rv = inflateEnd(rstream_); + checkZlibRvNothrow(rv, rstream_->msg); + } + // There is no way we can get here if wstream_ was initialized. + + throw; + } +} + +inline void TZlibTransport::checkZlibRv(int status, const char* message) { + if (status != Z_OK) { + throw TZlibTransportException(status, message); + } +} + +inline void TZlibTransport::checkZlibRvNothrow(int status, const char* message) { + if (status != Z_OK) { + string output = "TZlibTransport: zlib failure in destructor: " + + TZlibTransportException::errorMessage(status, message); + GlobalOutput(output.c_str()); + } +} + +TZlibTransport::~TZlibTransport() { + int rv; + rv = inflateEnd(rstream_); + checkZlibRvNothrow(rv, rstream_->msg); + rv = deflateEnd(wstream_); + checkZlibRvNothrow(rv, wstream_->msg); + + delete[] urbuf_; + delete[] crbuf_; + delete[] uwbuf_; + delete[] cwbuf_; + delete rstream_; + delete wstream_; +} + +bool TZlibTransport::isOpen() { + return (readAvail() > 0) || transport_->isOpen(); +} + +// READING STRATEGY +// +// We have two buffers for reading: one containing the compressed data (crbuf_) +// and one containing the uncompressed data (urbuf_). When read is called, +// we repeat the following steps until we have satisfied the request: +// - Copy data from urbuf_ into the caller's buffer. +// - If we had enough, return. +// - If urbuf_ is empty, read some data into it from the underlying transport. +// - Inflate data from crbuf_ into urbuf_. +// +// In standalone objects, we set input_ended_ to true when inflate returns +// Z_STREAM_END. This allows to make sure that a checksum was verified. + +inline int TZlibTransport::readAvail() { + return urbuf_size_ - rstream_->avail_out - urpos_; +} + +uint32_t TZlibTransport::read(uint8_t* buf, uint32_t len) { + int need = len; + + // TODO(dreiss): Skip urbuf on big reads. + + while (true) { + // Copy out whatever we have available, then give them the min of + // what we have and what they want, then advance indices. + int give = std::min(readAvail(), need); + memcpy(buf, urbuf_ + urpos_, give); + need -= give; + buf += give; + urpos_ += give; + + // If they were satisfied, we are done. + if (need == 0) { + return len; + } + + // If we get to this point, we need to get some more data. + + // If zlib has reported the end of a stream, we can't really do any more. + if (input_ended_) { + return len - need; + } + + // The uncompressed read buffer is empty, so reset the stream fields. + rstream_->next_out = urbuf_; + rstream_->avail_out = urbuf_size_; + urpos_ = 0; + + // If we don't have any more compressed data available, + // read some from the underlying transport. + if (rstream_->avail_in == 0) { + uint32_t got = transport_->read(crbuf_, crbuf_size_); + if (got == 0) { + return len - need; + } + rstream_->next_in = crbuf_; + rstream_->avail_in = got; + } + + // We have some compressed data now. Uncompress it. + int zlib_rv = inflate(rstream_, Z_SYNC_FLUSH); + + if (zlib_rv == Z_STREAM_END) { + if (standalone_) { + input_ended_ = true; + } + } else { + checkZlibRv(zlib_rv, rstream_->msg); + } + + // Okay. The read buffer should have whatever we can give it now. + // Loop back to the start and try to give some more. + } +} + + +// WRITING STRATEGY +// +// We buffer up small writes before sending them to zlib, so our logic is: +// - Is the write big? +// - Send the buffer to zlib. +// - Send this data to zlib. +// - Is the write small? +// - Is there insufficient space in the buffer for it? +// - Send the buffer to zlib. +// - Copy the data to the buffer. +// +// We have two buffers for writing also: the uncompressed buffer (mentioned +// above) and the compressed buffer. When sending data to zlib we loop over +// the following until the source (uncompressed buffer or big write) is empty: +// - Is there no more space in the compressed buffer? +// - Write the compressed buffer to the underlying transport. +// - Deflate from the source into the compressed buffer. + +void TZlibTransport::write(const uint8_t* buf, uint32_t len) { + // zlib's "deflate" function has enough logic in it that I think + // we're better off (performance-wise) buffering up small writes. + if ((int)len > MIN_DIRECT_DEFLATE_SIZE) { + flushToZlib(uwbuf_, uwpos_); + uwpos_ = 0; + flushToZlib(buf, len); + } else if (len > 0) { + if (uwbuf_size_ - uwpos_ < (int)len) { + flushToZlib(uwbuf_, uwpos_); + uwpos_ = 0; + } + memcpy(uwbuf_ + uwpos_, buf, len); + uwpos_ += len; + } +} + +void TZlibTransport::flush() { + flushToZlib(uwbuf_, uwpos_, true); + assert((int)wstream_->avail_out != cwbuf_size_); + transport_->write(cwbuf_, cwbuf_size_ - wstream_->avail_out); + transport_->flush(); +} + +void TZlibTransport::flushToZlib(const uint8_t* buf, int len, bool finish) { + int flush = (finish ? Z_FINISH : Z_NO_FLUSH); + + wstream_->next_in = const_cast(buf); + wstream_->avail_in = len; + + while (wstream_->avail_in > 0 || finish) { + // If our ouput buffer is full, flush to the underlying transport. + if (wstream_->avail_out == 0) { + transport_->write(cwbuf_, cwbuf_size_); + wstream_->next_out = cwbuf_; + wstream_->avail_out = cwbuf_size_; + } + + int zlib_rv = deflate(wstream_, flush); + + if (finish && zlib_rv == Z_STREAM_END) { + assert(wstream_->avail_in == 0); + break; + } + + checkZlibRv(zlib_rv, wstream_->msg); + } +} + +const uint8_t* TZlibTransport::borrow(uint8_t* buf, uint32_t* len) { + // Don't try to be clever with shifting buffers. + // If we have enough data, give a pointer to it, + // otherwise let the protcol use its slow path. + if (readAvail() >= (int)*len) { + *len = (uint32_t)readAvail(); + return urbuf_ + urpos_; + } + return NULL; +} + +void TZlibTransport::consume(uint32_t len) { + if (readAvail() >= (int)len) { + urpos_ += len; + } else { + throw TTransportException(TTransportException::BAD_ARGS, + "consume did not follow a borrow."); + } +} + +void TZlibTransport::verifyChecksum() { + if (!standalone_) { + throw TTransportException( + TTransportException::BAD_ARGS, + "TZLibTransport can only verify checksums for standalone objects."); + } + + if (!input_ended_) { + // This should only be called when reading is complete, + // but it's possible that the whole checksum has not been fed to zlib yet. + // We try to read an extra byte here to force zlib to finish the stream. + // It might not always be easy to "unread" this byte, + // but we throw an exception if we get it, which is not really + // a recoverable error, so it doesn't matter. + uint8_t buf[1]; + uint32_t got = this->read(buf, sizeof(buf)); + if (got || !input_ended_) { + throw TTransportException( + TTransportException::CORRUPTED_DATA, + "Zlib stream not complete."); + } + } + + // If the checksum had been bad, we would have gotten an error while + // inflating. +} + + +}}} // apache::thrift::transport diff --git a/lib/cpp/src/transport/TZlibTransport.h b/lib/cpp/src/transport/TZlibTransport.h new file mode 100644 index 00000000..1439d9de --- /dev/null +++ b/lib/cpp/src/transport/TZlibTransport.h @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TRANSPORT_TZLIBTRANSPORT_H_ +#define _THRIFT_TRANSPORT_TZLIBTRANSPORT_H_ 1 + +#include +#include + +struct z_stream_s; + +namespace apache { namespace thrift { namespace transport { + +class TZlibTransportException : public TTransportException { + public: + TZlibTransportException(int status, const char* msg) : + TTransportException(TTransportException::INTERNAL_ERROR, + errorMessage(status, msg)), + zlib_status_(status), + zlib_msg_(msg == NULL ? "(null)" : msg) {} + + virtual ~TZlibTransportException() throw() {} + + int getZlibStatus() { return zlib_status_; } + std::string getZlibMessage() { return zlib_msg_; } + + static std::string errorMessage(int status, const char* msg) { + std::string rv = "zlib error: "; + if (msg) { + rv += msg; + } else { + rv += "(no message)"; + } + rv += " (status = "; + rv += boost::lexical_cast(status); + rv += ")"; + return rv; + } + + int zlib_status_; + std::string zlib_msg_; +}; + +/** + * This transport uses zlib's compressed format on the "far" side. + * + * There are two kinds of TZlibTransport objects: + * - Standalone objects are used to encode self-contained chunks of data + * (like structures). They include checksums. + * - Non-standalone transports are used for RPC. They are not implemented yet. + * + * TODO(dreiss): Don't do an extra copy of the compressed data if + * the underlying transport is TBuffered or TMemory. + * + */ +class TZlibTransport : public TTransport { + public: + + /** + * @param transport The transport to read compressed data from + * and write compressed data to. + * @param use_for_rpc True if this object will be used for RPC, + * false if this is a standalone object. + * @param urbuf_size Uncompressed buffer size for reading. + * @param crbuf_size Compressed buffer size for reading. + * @param uwbuf_size Uncompressed buffer size for writing. + * @param cwbuf_size Compressed buffer size for writing. + * + * TODO(dreiss): Write a constructor that isn't a pain. + */ + TZlibTransport(boost::shared_ptr transport, + bool use_for_rpc, + int urbuf_size = DEFAULT_URBUF_SIZE, + int crbuf_size = DEFAULT_CRBUF_SIZE, + int uwbuf_size = DEFAULT_UWBUF_SIZE, + int cwbuf_size = DEFAULT_CWBUF_SIZE) : + transport_(transport), + standalone_(!use_for_rpc), + urpos_(0), + uwpos_(0), + input_ended_(false), + output_flushed_(false), + urbuf_size_(urbuf_size), + crbuf_size_(crbuf_size), + uwbuf_size_(uwbuf_size), + cwbuf_size_(cwbuf_size), + urbuf_(NULL), + crbuf_(NULL), + uwbuf_(NULL), + cwbuf_(NULL), + rstream_(NULL), + wstream_(NULL) + { + + if (!standalone_) { + throw TTransportException( + TTransportException::BAD_ARGS, + "TZLibTransport has not been tested for RPC."); + } + + if (uwbuf_size_ < MIN_DIRECT_DEFLATE_SIZE) { + // Have to copy this into a local because of a linking issue. + int minimum = MIN_DIRECT_DEFLATE_SIZE; + throw TTransportException( + TTransportException::BAD_ARGS, + "TZLibTransport: uncompressed write buffer must be at least" + + boost::lexical_cast(minimum) + "."); + } + + try { + urbuf_ = new uint8_t[urbuf_size]; + crbuf_ = new uint8_t[crbuf_size]; + uwbuf_ = new uint8_t[uwbuf_size]; + cwbuf_ = new uint8_t[cwbuf_size]; + + // Don't call this outside of the constructor. + initZlib(); + + } catch (...) { + delete[] urbuf_; + delete[] crbuf_; + delete[] uwbuf_; + delete[] cwbuf_; + throw; + } + } + + // Don't call this outside of the constructor. + void initZlib(); + + ~TZlibTransport(); + + bool isOpen(); + + void open() { + transport_->open(); + } + + void close() { + transport_->close(); + } + + uint32_t read(uint8_t* buf, uint32_t len); + + void write(const uint8_t* buf, uint32_t len); + + void flush(); + + const uint8_t* borrow(uint8_t* buf, uint32_t* len); + + void consume(uint32_t len); + + void verifyChecksum(); + + /** + * TODO(someone_smart): Choose smart defaults. + */ + static const int DEFAULT_URBUF_SIZE = 128; + static const int DEFAULT_CRBUF_SIZE = 1024; + static const int DEFAULT_UWBUF_SIZE = 128; + static const int DEFAULT_CWBUF_SIZE = 1024; + + protected: + + inline void checkZlibRv(int status, const char* msg); + inline void checkZlibRvNothrow(int status, const char* msg); + inline int readAvail(); + void flushToZlib(const uint8_t* buf, int len, bool finish = false); + + // Writes smaller than this are buffered up. + // Larger (or equal) writes are dumped straight to zlib. + static const int MIN_DIRECT_DEFLATE_SIZE = 32; + + boost::shared_ptr transport_; + bool standalone_; + + int urpos_; + int uwpos_; + + /// True iff zlib has reached the end of a stream. + /// This is only ever true in standalone protcol objects. + bool input_ended_; + /// True iff we have flushed the output stream. + /// This is only ever true in standalone protcol objects. + bool output_flushed_; + + int urbuf_size_; + int crbuf_size_; + int uwbuf_size_; + int cwbuf_size_; + + uint8_t* urbuf_; + uint8_t* crbuf_; + uint8_t* uwbuf_; + uint8_t* cwbuf_; + + struct z_stream_s* rstream_; + struct z_stream_s* wstream_; +}; + +}}} // apache::thrift::transport + +#endif // #ifndef _THRIFT_TRANSPORT_TZLIBTRANSPORT_H_ diff --git a/lib/cpp/thrift-nb.pc.in b/lib/cpp/thrift-nb.pc.in new file mode 100644 index 00000000..ae051887 --- /dev/null +++ b/lib/cpp/thrift-nb.pc.in @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Thrift +Description: Thrift Nonblocking API +Version: @VERSION@ +Requires: thrift = @VERSION@ +Libs: -L${libdir} -lthriftnb +Cflags: -I${includedir}/thrift diff --git a/lib/cpp/thrift-z.pc.in b/lib/cpp/thrift-z.pc.in new file mode 100644 index 00000000..72f46bf9 --- /dev/null +++ b/lib/cpp/thrift-z.pc.in @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Thrift +Description: Thrift Zlib API +Version: @VERSION@ +Requires: thrift = @VERSION@ +Libs: -L${libdir} -lthriftz +Cflags: -I${includedir}/thrift diff --git a/lib/cpp/thrift.pc.in b/lib/cpp/thrift.pc.in new file mode 100644 index 00000000..7aec09f1 --- /dev/null +++ b/lib/cpp/thrift.pc.in @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Thrift +Description: Thrift C++ API +Version: @VERSION@ +Libs: -L${libdir} -lthrift +Cflags: -I${includedir}/thrift diff --git a/lib/csharp/Makefile.am b/lib/csharp/Makefile.am new file mode 100644 index 00000000..4047011c --- /dev/null +++ b/lib/csharp/Makefile.am @@ -0,0 +1,70 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +THRIFTCODE= \ + src/Collections/THashSet.cs \ + src/Protocol/TBase.cs \ + src/Protocol/TProtocolException.cs \ + src/Protocol/TProtocolFactory.cs \ + src/Protocol/TList.cs \ + src/Protocol/TSet.cs \ + src/Protocol/TMap.cs \ + src/Protocol/TProtocolUtil.cs \ + src/Protocol/TMessageType.cs \ + src/Protocol/TProtocol.cs \ + src/Protocol/TType.cs \ + src/Protocol/TField.cs \ + src/Protocol/TMessage.cs \ + src/Protocol/TStruct.cs \ + src/Protocol/TBinaryProtocol.cs \ + src/Server/TThreadedServer.cs \ + src/Server/TThreadPoolServer.cs \ + src/Server/TSimpleServer.cs \ + src/Server/TServer.cs \ + src/Transport/TBufferedTransport.cs \ + src/Transport/TTransport.cs \ + src/Transport/TSocket.cs \ + src/Transport/TTransportException.cs \ + src/Transport/TStreamTransport.cs \ + src/Transport/TServerTransport.cs \ + src/Transport/TServerSocket.cs \ + src/Transport/TTransportFactory.cs \ + src/TProcessor.cs \ + src/TApplicationException.cs + + +CSC=gmcs + +if NET_2_0 +MONO_DEFINES=/d:NET_2_0 +endif + +all-local: Thrift.dll + +Thrift.dll: $(THRIFTCODE) + $(CSC) $(THRIFTCODE) /out:Thrift.dll /target:library $(MONO_DEFINES) + +clean-local: + $(RM) Thrift.dll + +EXTRA_DIST = \ + $(THRIFTCODE) \ + ThriftMSBuildTask \ + src/Thrift.csproj \ + src/Thrift.sln diff --git a/lib/csharp/README b/lib/csharp/README new file mode 100644 index 00000000..b7dc5de3 --- /dev/null +++ b/lib/csharp/README @@ -0,0 +1,26 @@ +Thrift C# Software Library + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Using Thrift with C# +==================== + +Thrift requires Mono >= 1.2.6 or .NET framework >= 3.5 diff --git a/lib/csharp/ThriftMSBuildTask/Properties/AssemblyInfo.cs b/lib/csharp/ThriftMSBuildTask/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..d79c2039 --- /dev/null +++ b/lib/csharp/ThriftMSBuildTask/Properties/AssemblyInfo.cs @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("ThriftMSBuildTask")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("ThriftMSBuildTask")] +[assembly: AssemblyCopyright("Copyright © 2009 The Apache Software Foundation")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("5095e09d-7b95-4be1-b250-e1c1db1c485e")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/lib/csharp/ThriftMSBuildTask/ThriftBuild.cs b/lib/csharp/ThriftMSBuildTask/ThriftBuild.cs new file mode 100644 index 00000000..4389e0a6 --- /dev/null +++ b/lib/csharp/ThriftMSBuildTask/ThriftBuild.cs @@ -0,0 +1,242 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.Build.Tasks; +using System.IO; +using System.Diagnostics; + +namespace ThriftMSBuildTask +{ + /// + /// MSBuild Task to generate csharp from .thrift files, and compile the code into a library: ThriftImpl.dll + /// + public class ThriftBuild : Task + { + /// + /// The full path to the thrift.exe compiler + /// + [Required] + public ITaskItem ThriftExecutable + { + get; + set; + } + + /// + /// The full path to a thrift.dll C# library + /// + [Required] + public ITaskItem ThriftLibrary + { + get; + set; + } + + /// + /// A direcotry containing .thrift files + /// + [Required] + public ITaskItem ThriftDefinitionDir + { + get; + set; + } + + /// + /// The name of the auto-gen and compiled thrift library. It will placed in + /// the same directory as ThriftLibrary + /// + [Required] + public ITaskItem OutputName + { + get; + set; + } + + /// + /// The full path to the compiled ThriftLibrary. This allows msbuild tasks to use this + /// output as a variable for use elsewhere. + /// + [Output] + public ITaskItem ThriftImplementation + { + get { return thriftImpl; } + } + + private ITaskItem thriftImpl; + private const string lastCompilationName = "LAST_COMP_TIMESTAMP"; + + //use the Message Build Task to write something to build log + private void LogMessage(string text, MessageImportance importance) + { + Message m = new Message(); + m.Text = text; + m.Importance = importance.ToString(); + m.BuildEngine = this.BuildEngine; + m.Execute(); + } + + //recursively find .cs files in srcDir, paths should initially be non-null and empty + private void FindSourcesHelper(string srcDir, List paths) + { + string[] files = Directory.GetFiles(srcDir, "*.cs"); + foreach (string f in files) + { + paths.Add(f); + } + string[] dirs = Directory.GetDirectories(srcDir); + foreach (string dir in dirs) + { + FindSourcesHelper(dir, paths); + } + } + + /// + /// Quote paths with spaces + /// + private string SafePath(string path) + { + if (path.Contains(' ') && !path.StartsWith("\"")) + { + return "\"" + path + "\""; + } + return path; + } + + private ITaskItem[] FindSources(string srcDir) + { + List files = new List(); + FindSourcesHelper(srcDir, files); + ITaskItem[] items = new ITaskItem[files.Count]; + for (int i = 0; i < items.Length; i++) + { + items[i] = new TaskItem(files[i]); + } + return items; + } + + private string LastWriteTime(string defDir) + { + string[] files = Directory.GetFiles(defDir, "*.thrift"); + DateTime d = (new DirectoryInfo(defDir)).LastWriteTime; + foreach(string file in files) + { + FileInfo f = new FileInfo(file); + DateTime curr = f.LastWriteTime; + if (DateTime.Compare(curr, d) > 0) + { + d = curr; + } + } + return d.ToFileTimeUtc().ToString(); + } + + public override bool Execute() + { + string defDir = SafePath(ThriftDefinitionDir.ItemSpec); + //look for last compilation timestamp + string lastBuildPath = Path.Combine(defDir, lastCompilationName); + DirectoryInfo defDirInfo = new DirectoryInfo(defDir); + string lastWrite = LastWriteTime(defDir); + if (File.Exists(lastBuildPath)) + { + string lastComp = File.ReadAllText(lastBuildPath); + //don't recompile if the thrift library has been updated since lastComp + FileInfo f = new FileInfo(ThriftLibrary.ItemSpec); + string thriftLibTime = f.LastWriteTimeUtc.ToFileTimeUtc().ToString(); + if (lastComp.CompareTo(thriftLibTime) < 0) + { + //new thrift library, do a compile + lastWrite = thriftLibTime; + } + else if (lastComp == lastWrite || (lastComp == thriftLibTime && lastComp.CompareTo(lastWrite) > 0)) + { + //the .thrift dir hasn't been written to since last compilation, don't need to do anything + LogMessage("ThriftImpl up-to-date", MessageImportance.High); + return true; + } + } + + //find the directory of the thriftlibrary (that's where output will go) + FileInfo thriftLibInfo = new FileInfo(SafePath(ThriftLibrary.ItemSpec)); + string thriftDir = thriftLibInfo.Directory.FullName; + + string genDir = Path.Combine(thriftDir, "gen-csharp"); + if (Directory.Exists(genDir)) + { + try + { + Directory.Delete(genDir, true); + } + catch { /*eh i tried, just over-write now*/} + } + + //run the thrift executable to generate C# + foreach (string thriftFile in Directory.GetFiles(defDir, "*.thrift")) + { + LogMessage("Generating code for: " + thriftFile, MessageImportance.Normal); + Process p = new Process(); + p.StartInfo.FileName = SafePath(ThriftExecutable.ItemSpec); + p.StartInfo.Arguments = "--gen csharp -o " + SafePath(thriftDir) + " -r " + thriftFile; + p.StartInfo.UseShellExecute = false; + p.StartInfo.CreateNoWindow = true; + p.StartInfo.RedirectStandardOutput = false; + p.Start(); + p.WaitForExit(); + if (p.ExitCode != 0) + { + LogMessage("thrift.exe failed to compile " + thriftFile, MessageImportance.High); + return false; + } + if (p.ExitCode != 0) + { + LogMessage("thrift.exe failed to compile " + thriftFile, MessageImportance.High); + return false; + } + } + + Csc csc = new Csc(); + csc.TargetType = "library"; + csc.References = new ITaskItem[] { new TaskItem(ThriftLibrary.ItemSpec) }; + csc.EmitDebugInformation = true; + string outputPath = Path.Combine(thriftDir, OutputName.ItemSpec); + csc.OutputAssembly = new TaskItem(outputPath); + csc.Sources = FindSources(Path.Combine(thriftDir, "gen-csharp")); + csc.BuildEngine = this.BuildEngine; + LogMessage("Compiling generated cs...", MessageImportance.Normal); + if (!csc.Execute()) + { + return false; + } + + //write file to defDir to indicate a build was successfully completed + File.WriteAllText(lastBuildPath, lastWrite); + + thriftImpl = new TaskItem(outputPath); + + return true; + } + } +} diff --git a/lib/csharp/ThriftMSBuildTask/ThriftMSBuildTask.csproj b/lib/csharp/ThriftMSBuildTask/ThriftMSBuildTask.csproj new file mode 100644 index 00000000..02110eae --- /dev/null +++ b/lib/csharp/ThriftMSBuildTask/ThriftMSBuildTask.csproj @@ -0,0 +1,66 @@ + + + + Debug + AnyCPU + 9.0.21022 + 2.0 + {EC0A0231-66EA-4593-A792-C6CA3BB8668E} + Library + Properties + ThriftMSBuildTask + ThriftMSBuildTask + v3.5 + 512 + SAK + SAK + SAK + SAK + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + 3.5 + + + 3.5 + + + 3.5 + + + + + + + + + + + diff --git a/lib/csharp/src/Collections/THashSet.cs b/lib/csharp/src/Collections/THashSet.cs new file mode 100644 index 00000000..a9957693 --- /dev/null +++ b/lib/csharp/src/Collections/THashSet.cs @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections; +using System.Collections.Generic; + +namespace Thrift.Collections +{ + public class THashSet : ICollection + { +#if NET_2_0 + TDictSet set = new TDictSet(); +#else + HashSet set = new HashSet(); +#endif + public int Count + { + get { return set.Count; } + } + + public bool IsReadOnly + { + get { return false; } + } + + public void Add(T item) + { + set.Add(item); + } + + public void Clear() + { + set.Clear(); + } + + public bool Contains(T item) + { + return set.Contains(item); + } + + public void CopyTo(T[] array, int arrayIndex) + { + set.CopyTo(array, arrayIndex); + } + + public IEnumerator GetEnumerator() + { + return set.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return ((IEnumerable)set).GetEnumerator(); + } + + public bool Remove(T item) + { + return set.Remove(item); + } + +#if NET_2_0 + private class TDictSet : ICollection + { + Dictionary> dict = new Dictionary>(); + + public int Count + { + get { return dict.Count; } + } + + public bool IsReadOnly + { + get { return false; } + } + + public IEnumerator GetEnumerator() + { + return ((IEnumerable)dict.Keys).GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return dict.Keys.GetEnumerator(); + } + + public bool Add(V item) + { + if (!dict.ContainsKey(item)) + { + dict[item] = this; + return true; + } + + return false; + } + + void ICollection.Add(V item) + { + Add(item); + } + + public void Clear() + { + dict.Clear(); + } + + public bool Contains(V item) + { + return dict.ContainsKey(item); + } + + public void CopyTo(V[] array, int arrayIndex) + { + dict.Keys.CopyTo(array, arrayIndex); + } + + public bool Remove(V item) + { + return dict.Remove(item); + } + } +#endif + } + +} diff --git a/lib/csharp/src/Protocol/TBase.cs b/lib/csharp/src/Protocol/TBase.cs new file mode 100644 index 00000000..1969bb3d --- /dev/null +++ b/lib/csharp/src/Protocol/TBase.cs @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace Thrift.Protocol +{ + public interface TBase + { + /// + /// Reads the TObject from the given input protocol. + /// + void Read(TProtocol tProtocol); + + /// + /// Writes the objects out to the protocol + /// + void Write(TProtocol tProtocol); + } +} diff --git a/lib/csharp/src/Protocol/TBinaryProtocol.cs b/lib/csharp/src/Protocol/TBinaryProtocol.cs new file mode 100644 index 00000000..14ca43b7 --- /dev/null +++ b/lib/csharp/src/Protocol/TBinaryProtocol.cs @@ -0,0 +1,392 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Text; +using Thrift.Transport; + +namespace Thrift.Protocol +{ + public class TBinaryProtocol : TProtocol + { + protected const uint VERSION_MASK = 0xffff0000; + protected const uint VERSION_1 = 0x80010000; + + protected bool strictRead_ = false; + protected bool strictWrite_ = true; + + protected int readLength_; + protected bool checkReadLength_ = false; + + + #region BinaryProtocol Factory + /** + * Factory + */ + public class Factory : TProtocolFactory { + + protected bool strictRead_ = false; + protected bool strictWrite_ = true; + + public Factory() + :this(false, true) + { + } + + public Factory(bool strictRead, bool strictWrite) + { + strictRead_ = strictRead; + strictWrite_ = strictWrite; + } + + public TProtocol GetProtocol(TTransport trans) { + return new TBinaryProtocol(trans, strictRead_, strictWrite_); + } + } + + #endregion + + public TBinaryProtocol(TTransport trans) + : this(trans, false, true) + { + } + + public TBinaryProtocol(TTransport trans, bool strictRead, bool strictWrite) + :base(trans) + { + strictRead_ = strictRead; + strictWrite_ = strictWrite; + } + + #region Write Methods + + public override void WriteMessageBegin(TMessage message) + { + if (strictWrite_) + { + uint version = VERSION_1 | (uint)(message.Type); + WriteI32((int)version); + WriteString(message.Name); + WriteI32(message.SeqID); + } + else + { + WriteString(message.Name); + WriteByte((byte)message.Type); + WriteI32(message.SeqID); + } + } + + public override void WriteMessageEnd() + { + } + + public override void WriteStructBegin(TStruct struc) + { + } + + public override void WriteStructEnd() + { + } + + public override void WriteFieldBegin(TField field) + { + WriteByte((byte)field.Type); + WriteI16(field.ID); + } + + public override void WriteFieldEnd() + { + } + + public override void WriteFieldStop() + { + WriteByte((byte)TType.Stop); + } + + public override void WriteMapBegin(TMap map) + { + WriteByte((byte)map.KeyType); + WriteByte((byte)map.ValueType); + WriteI32(map.Count); + } + + public override void WriteMapEnd() + { + } + + public override void WriteListBegin(TList list) + { + WriteByte((byte)list.ElementType); + WriteI32(list.Count); + } + + public override void WriteListEnd() + { + } + + public override void WriteSetBegin(TSet set) + { + WriteByte((byte)set.ElementType); + WriteI32(set.Count); + } + + public override void WriteSetEnd() + { + } + + public override void WriteBool(bool b) + { + WriteByte(b ? (byte)1 : (byte)0); + } + + private byte[] bout = new byte[1]; + public override void WriteByte(byte b) + { + bout[0] = b; + trans.Write(bout, 0, 1); + } + + private byte[] i16out = new byte[2]; + public override void WriteI16(short s) + { + i16out[0] = (byte)(0xff & (s >> 8)); + i16out[1] = (byte)(0xff & s); + trans.Write(i16out, 0, 2); + } + + private byte[] i32out = new byte[4]; + public override void WriteI32(int i32) + { + i32out[0] = (byte)(0xff & (i32 >> 24)); + i32out[1] = (byte)(0xff & (i32 >> 16)); + i32out[2] = (byte)(0xff & (i32 >> 8)); + i32out[3] = (byte)(0xff & i32); + trans.Write(i32out, 0, 4); + } + + private byte[] i64out = new byte[8]; + public override void WriteI64(long i64) + { + i64out[0] = (byte)(0xff & (i64 >> 56)); + i64out[1] = (byte)(0xff & (i64 >> 48)); + i64out[2] = (byte)(0xff & (i64 >> 40)); + i64out[3] = (byte)(0xff & (i64 >> 32)); + i64out[4] = (byte)(0xff & (i64 >> 24)); + i64out[5] = (byte)(0xff & (i64 >> 16)); + i64out[6] = (byte)(0xff & (i64 >> 8)); + i64out[7] = (byte)(0xff & i64); + trans.Write(i64out, 0, 8); + } + + public override void WriteDouble(double d) + { + WriteI64(BitConverter.DoubleToInt64Bits(d)); + } + + public override void WriteBinary(byte[] b) + { + WriteI32(b.Length); + trans.Write(b, 0, b.Length); + } + + #endregion + + #region ReadMethods + + public override TMessage ReadMessageBegin() + { + TMessage message = new TMessage(); + int size = ReadI32(); + if (size < 0) + { + uint version = (uint)size & VERSION_MASK; + if (version != VERSION_1) + { + throw new TProtocolException(TProtocolException.BAD_VERSION, "Bad version in ReadMessageBegin: " + version); + } + message.Type = (TMessageType)(size & 0x000000ff); + message.Name = ReadString(); + message.SeqID = ReadI32(); + } + else + { + if (strictRead_) + { + throw new TProtocolException(TProtocolException.BAD_VERSION, "Missing version in readMessageBegin, old client?"); + } + message.Name = ReadStringBody(size); + message.Type = (TMessageType)ReadByte(); + message.SeqID = ReadI32(); + } + return message; + } + + public override void ReadMessageEnd() + { + } + + public override TStruct ReadStructBegin() + { + return new TStruct(); + } + + public override void ReadStructEnd() + { + } + + public override TField ReadFieldBegin() + { + TField field = new TField(); + field.Type = (TType)ReadByte(); + + if (field.Type != TType.Stop) + { + field.ID = ReadI16(); + } + + return field; + } + + public override void ReadFieldEnd() + { + } + + public override TMap ReadMapBegin() + { + TMap map = new TMap(); + map.KeyType = (TType)ReadByte(); + map.ValueType = (TType)ReadByte(); + map.Count = ReadI32(); + + return map; + } + + public override void ReadMapEnd() + { + } + + public override TList ReadListBegin() + { + TList list = new TList(); + list.ElementType = (TType)ReadByte(); + list.Count = ReadI32(); + + return list; + } + + public override void ReadListEnd() + { + } + + public override TSet ReadSetBegin() + { + TSet set = new TSet(); + set.ElementType = (TType)ReadByte(); + set.Count = ReadI32(); + + return set; + } + + public override void ReadSetEnd() + { + } + + public override bool ReadBool() + { + return ReadByte() == 1; + } + + private byte[] bin = new byte[1]; + public override byte ReadByte() + { + ReadAll(bin, 0, 1); + return bin[0]; + } + + private byte[] i16in = new byte[2]; + public override short ReadI16() + { + ReadAll(i16in, 0, 2); + return (short)(((i16in[0] & 0xff) << 8) | ((i16in[1] & 0xff))); + } + + private byte[] i32in = new byte[4]; + public override int ReadI32() + { + ReadAll(i32in, 0, 4); + return (int)(((i32in[0] & 0xff) << 24) | ((i32in[1] & 0xff) << 16) | ((i32in[2] & 0xff) << 8) | ((i32in[3] & 0xff))); + } + + private byte[] i64in = new byte[8]; + public override long ReadI64() + { + ReadAll(i64in, 0, 8); + return (long)(((long)(i64in[0] & 0xff) << 56) | ((long)(i64in[1] & 0xff) << 48) | ((long)(i64in[2] & 0xff) << 40) | ((long)(i64in[3] & 0xff) << 32) | + ((long)(i64in[4] & 0xff) << 24) | ((long)(i64in[5] & 0xff) << 16) | ((long)(i64in[6] & 0xff) << 8) | ((long)(i64in[7] & 0xff))); + } + + public override double ReadDouble() + { + return BitConverter.Int64BitsToDouble(ReadI64()); + } + + public void SetReadLength(int readLength) + { + readLength_ = readLength; + checkReadLength_ = true; + } + + protected void CheckReadLength(int length) + { + if (checkReadLength_) + { + readLength_ -= length; + if (readLength_ < 0) + { + throw new Exception("Message length exceeded: " + length); + } + } + } + + public override byte[] ReadBinary() + { + int size = ReadI32(); + CheckReadLength(size); + byte[] buf = new byte[size]; + trans.ReadAll(buf, 0, size); + return buf; + } + private string ReadStringBody(int size) + { + CheckReadLength(size); + byte[] buf = new byte[size]; + trans.ReadAll(buf, 0, size); + return Encoding.UTF8.GetString(buf); + } + + private int ReadAll(byte[] buf, int off, int len) + { + CheckReadLength(len); + return trans.ReadAll(buf, off, len); + } + + #endregion + } +} diff --git a/lib/csharp/src/Protocol/TField.cs b/lib/csharp/src/Protocol/TField.cs new file mode 100644 index 00000000..485c994b --- /dev/null +++ b/lib/csharp/src/Protocol/TField.cs @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Thrift.Protocol +{ + public struct TField + { + private string name; + private TType type; + private short id; + + public TField(string name, TType type, short id) + :this() + { + this.name = name; + this.type = type; + this.id = id; + } + + public string Name + { + get { return name; } + set { name = value; } + } + + public TType Type + { + get { return type; } + set { type = value; } + } + + public short ID + { + get { return id; } + set { id = value; } + } + } +} diff --git a/lib/csharp/src/Protocol/TList.cs b/lib/csharp/src/Protocol/TList.cs new file mode 100644 index 00000000..dbc5c40e --- /dev/null +++ b/lib/csharp/src/Protocol/TList.cs @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Thrift.Protocol +{ + public struct TList + { + private TType elementType; + private int count; + + public TList(TType elementType, int count) + :this() + { + this.elementType = elementType; + this.count = count; + } + + public TType ElementType + { + get { return elementType; } + set { elementType = value; } + } + + public int Count + { + get { return count; } + set { count = value; } + } + } +} diff --git a/lib/csharp/src/Protocol/TMap.cs b/lib/csharp/src/Protocol/TMap.cs new file mode 100644 index 00000000..8b53f899 --- /dev/null +++ b/lib/csharp/src/Protocol/TMap.cs @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Thrift.Protocol +{ + public struct TMap + { + private TType keyType; + private TType valueType; + private int count; + + public TMap(TType keyType, TType valueType, int count) + :this() + { + this.keyType = keyType; + this.valueType = valueType; + this.count = count; + } + + public TType KeyType + { + get { return keyType; } + set { keyType = value; } + } + + public TType ValueType + { + get { return valueType; } + set { valueType = value; } + } + + public int Count + { + get { return count; } + set { count = value; } + } + } +} diff --git a/lib/csharp/src/Protocol/TMessage.cs b/lib/csharp/src/Protocol/TMessage.cs new file mode 100644 index 00000000..8cb6e0b1 --- /dev/null +++ b/lib/csharp/src/Protocol/TMessage.cs @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Thrift.Protocol +{ + public struct TMessage + { + private string name; + private TMessageType type; + private int seqID; + + public TMessage(string name, TMessageType type, int seqid) + :this() + { + this.name = name; + this.type = type; + this.seqID = seqid; + } + + public string Name + { + get { return name; } + set { name = value; } + } + + public TMessageType Type + { + get { return type; } + set { type = value; } + } + + public int SeqID + { + get { return seqID; } + set { seqID = value; } + } + } +} diff --git a/lib/csharp/src/Protocol/TMessageType.cs b/lib/csharp/src/Protocol/TMessageType.cs new file mode 100644 index 00000000..ab07cf6c --- /dev/null +++ b/lib/csharp/src/Protocol/TMessageType.cs @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; + +namespace Thrift.Protocol +{ + public enum TMessageType + { + Call = 1, + Reply = 2, + Exception = 3, + Oneway = 4 + } +} diff --git a/lib/csharp/src/Protocol/TProtocol.cs b/lib/csharp/src/Protocol/TProtocol.cs new file mode 100644 index 00000000..acf9c1b3 --- /dev/null +++ b/lib/csharp/src/Protocol/TProtocol.cs @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Text; +using Thrift.Transport; + +namespace Thrift.Protocol +{ + public abstract class TProtocol + { + protected TTransport trans; + + protected TProtocol(TTransport trans) + { + this.trans = trans; + } + + public TTransport Transport + { + get { return trans; } + } + + public abstract void WriteMessageBegin(TMessage message); + public abstract void WriteMessageEnd(); + public abstract void WriteStructBegin(TStruct struc); + public abstract void WriteStructEnd(); + public abstract void WriteFieldBegin(TField field); + public abstract void WriteFieldEnd(); + public abstract void WriteFieldStop(); + public abstract void WriteMapBegin(TMap map); + public abstract void WriteMapEnd(); + public abstract void WriteListBegin(TList list); + public abstract void WriteListEnd(); + public abstract void WriteSetBegin(TSet set); + public abstract void WriteSetEnd(); + public abstract void WriteBool(bool b); + public abstract void WriteByte(byte b); + public abstract void WriteI16(short i16); + public abstract void WriteI32(int i32); + public abstract void WriteI64(long i64); + public abstract void WriteDouble(double d); + public void WriteString(string s) { + WriteBinary(Encoding.UTF8.GetBytes(s)); + } + public abstract void WriteBinary(byte[] b); + + public abstract TMessage ReadMessageBegin(); + public abstract void ReadMessageEnd(); + public abstract TStruct ReadStructBegin(); + public abstract void ReadStructEnd(); + public abstract TField ReadFieldBegin(); + public abstract void ReadFieldEnd(); + public abstract TMap ReadMapBegin(); + public abstract void ReadMapEnd(); + public abstract TList ReadListBegin(); + public abstract void ReadListEnd(); + public abstract TSet ReadSetBegin(); + public abstract void ReadSetEnd(); + public abstract bool ReadBool(); + public abstract byte ReadByte(); + public abstract short ReadI16(); + public abstract int ReadI32(); + public abstract long ReadI64(); + public abstract double ReadDouble(); + public string ReadString() { + return Encoding.UTF8.GetString(ReadBinary()); + } + public abstract byte[] ReadBinary(); + } +} diff --git a/lib/csharp/src/Protocol/TProtocolException.cs b/lib/csharp/src/Protocol/TProtocolException.cs new file mode 100644 index 00000000..9c250476 --- /dev/null +++ b/lib/csharp/src/Protocol/TProtocolException.cs @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; + +namespace Thrift.Protocol +{ + class TProtocolException : Exception + { + public const int UNKNOWN = 0; + public const int INVALID_DATA = 1; + public const int NEGATIVE_SIZE = 2; + public const int SIZE_LIMIT = 3; + public const int BAD_VERSION = 4; + + protected int type_ = UNKNOWN; + + public TProtocolException() + : base() + { + } + + public TProtocolException(int type) + : base() + { + type_ = type; + } + + public TProtocolException(int type, String message) + : base(message) + { + type_ = type; + } + + public TProtocolException(String message) + : base(message) + { + } + + public int getType() + { + return type_; + } + } +} diff --git a/lib/csharp/src/Protocol/TProtocolFactory.cs b/lib/csharp/src/Protocol/TProtocolFactory.cs new file mode 100644 index 00000000..ae976acd --- /dev/null +++ b/lib/csharp/src/Protocol/TProtocolFactory.cs @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using Thrift.Transport; + +namespace Thrift.Protocol +{ + public interface TProtocolFactory + { + TProtocol GetProtocol(TTransport trans); + } +} diff --git a/lib/csharp/src/Protocol/TProtocolUtil.cs b/lib/csharp/src/Protocol/TProtocolUtil.cs new file mode 100644 index 00000000..57cef0ef --- /dev/null +++ b/lib/csharp/src/Protocol/TProtocolUtil.cs @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; + +namespace Thrift.Protocol +{ + public static class TProtocolUtil + { + public static void Skip(TProtocol prot, TType type) + { + switch (type) + { + case TType.Bool: + prot.ReadBool(); + break; + case TType.Byte: + prot.ReadByte(); + break; + case TType.I16: + prot.ReadI16(); + break; + case TType.I32: + prot.ReadI32(); + break; + case TType.I64: + prot.ReadI64(); + break; + case TType.Double: + prot.ReadDouble(); + break; + case TType.String: + // Don't try to decode the string, just skip it. + prot.ReadBinary(); + break; + case TType.Struct: + prot.ReadStructBegin(); + while (true) + { + TField field = prot.ReadFieldBegin(); + if (field.Type == TType.Stop) + { + break; + } + Skip(prot, field.Type); + prot.ReadFieldEnd(); + } + prot.ReadStructEnd(); + break; + case TType.Map: + TMap map = prot.ReadMapBegin(); + for (int i = 0; i < map.Count; i++) + { + Skip(prot, map.KeyType); + Skip(prot, map.ValueType); + } + prot.ReadMapEnd(); + break; + case TType.Set: + TSet set = prot.ReadSetBegin(); + for (int i = 0; i < set.Count; i++) + { + Skip(prot, set.ElementType); + } + prot.ReadSetEnd(); + break; + case TType.List: + TList list = prot.ReadListBegin(); + for (int i = 0; i < list.Count; i++) + { + Skip(prot, list.ElementType); + } + prot.ReadListEnd(); + break; + } + } + } +} diff --git a/lib/csharp/src/Protocol/TSet.cs b/lib/csharp/src/Protocol/TSet.cs new file mode 100644 index 00000000..ac73992d --- /dev/null +++ b/lib/csharp/src/Protocol/TSet.cs @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Thrift.Protocol +{ + public struct TSet + { + private TType elementType; + private int count; + + public TSet(TType elementType, int count) + :this() + { + this.elementType = elementType; + this.count = count; + } + + public TType ElementType + { + get { return elementType; } + set { elementType = value; } + } + + public int Count + { + get { return count; } + set { count = value; } + } + } +} diff --git a/lib/csharp/src/Protocol/TStruct.cs b/lib/csharp/src/Protocol/TStruct.cs new file mode 100644 index 00000000..0cac2733 --- /dev/null +++ b/lib/csharp/src/Protocol/TStruct.cs @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Thrift.Protocol +{ + public struct TStruct + { + private string name; + + public TStruct(string name) + :this() + { + this.name = name; + } + + public string Name + { + get { return name; } + set { name = value; } + } + } +} diff --git a/lib/csharp/src/Protocol/TType.cs b/lib/csharp/src/Protocol/TType.cs new file mode 100644 index 00000000..c2d78edc --- /dev/null +++ b/lib/csharp/src/Protocol/TType.cs @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; + +namespace Thrift.Protocol +{ + public enum TType : byte + { + Stop = 0, + Void = 1, + Bool = 2, + Byte = 3, + Double = 4, + I16 = 6, + I32 = 8, + I64 = 10, + String = 11, + Struct = 12, + Map = 13, + Set = 14, + List = 15 + } +} diff --git a/lib/csharp/src/Server/TServer.cs b/lib/csharp/src/Server/TServer.cs new file mode 100644 index 00000000..61a9416f --- /dev/null +++ b/lib/csharp/src/Server/TServer.cs @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using Thrift.Protocol; +using Thrift.Transport; +using System.IO; + +namespace Thrift.Server +{ + public abstract class TServer + { + /** + * Core processor + */ + protected TProcessor processor; + + /** + * Server transport + */ + protected TServerTransport serverTransport; + + /** + * Input Transport Factory + */ + protected TTransportFactory inputTransportFactory; + + /** + * Output Transport Factory + */ + protected TTransportFactory outputTransportFactory; + + /** + * Input Protocol Factory + */ + protected TProtocolFactory inputProtocolFactory; + + /** + * Output Protocol Factory + */ + protected TProtocolFactory outputProtocolFactory; + public delegate void LogDelegate(string str); + protected LogDelegate logDelegate; + + /** + * Default constructors. + */ + + public TServer(TProcessor processor, + TServerTransport serverTransport) + :this(processor, serverTransport, new TTransportFactory(), new TTransportFactory(), new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory(), DefaultLogDelegate) + { + } + + public TServer(TProcessor processor, + TServerTransport serverTransport, + LogDelegate logDelegate) + : this(processor, serverTransport, new TTransportFactory(), new TTransportFactory(), new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory(), DefaultLogDelegate) + { + } + + public TServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory transportFactory) + :this(processor, + serverTransport, + transportFactory, + transportFactory, + new TBinaryProtocol.Factory(), + new TBinaryProtocol.Factory(), + DefaultLogDelegate) + { + } + + public TServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory transportFactory, + TProtocolFactory protocolFactory) + :this(processor, + serverTransport, + transportFactory, + transportFactory, + protocolFactory, + protocolFactory, + DefaultLogDelegate) + { + } + + public TServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory inputTransportFactory, + TTransportFactory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory, + LogDelegate logDelegate) + { + this.processor = processor; + this.serverTransport = serverTransport; + this.inputTransportFactory = inputTransportFactory; + this.outputTransportFactory = outputTransportFactory; + this.inputProtocolFactory = inputProtocolFactory; + this.outputProtocolFactory = outputProtocolFactory; + this.logDelegate = logDelegate; + } + + /** + * The run method fires up the server and gets things going. + */ + public abstract void Serve(); + + public abstract void Stop(); + + protected static void DefaultLogDelegate(string s) + { + Console.Error.WriteLine(s); + } + } +} + diff --git a/lib/csharp/src/Server/TSimpleServer.cs b/lib/csharp/src/Server/TSimpleServer.cs new file mode 100644 index 00000000..34a51de4 --- /dev/null +++ b/lib/csharp/src/Server/TSimpleServer.cs @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using Thrift.Transport; +using Thrift.Protocol; + +namespace Thrift.Server +{ + /// + /// Simple single-threaded server for testing + /// + public class TSimpleServer : TServer + { + private bool stop = false; + + public TSimpleServer(TProcessor processor, + TServerTransport serverTransport) + :base(processor, serverTransport, new TTransportFactory(), new TTransportFactory(), new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory(), DefaultLogDelegate) + { + } + + public TSimpleServer(TProcessor processor, + TServerTransport serverTransport, + LogDelegate logDel) + : base(processor, serverTransport, new TTransportFactory(), new TTransportFactory(), new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory(), logDel) + { + } + + public TSimpleServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory transportFactory) + :base(processor, + serverTransport, + transportFactory, + transportFactory, + new TBinaryProtocol.Factory(), + new TBinaryProtocol.Factory(), + DefaultLogDelegate) + { + } + + public TSimpleServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory transportFactory, + TProtocolFactory protocolFactory) + :base(processor, + serverTransport, + transportFactory, + transportFactory, + protocolFactory, + protocolFactory, + DefaultLogDelegate) + { + } + + public override void Serve() + { + try + { + serverTransport.Listen(); + } + catch (TTransportException ttx) + { + logDelegate(ttx.ToString()); + return; + } + + while (!stop) + { + TTransport client = null; + TTransport inputTransport = null; + TTransport outputTransport = null; + TProtocol inputProtocol = null; + TProtocol outputProtocol = null; + try + { + client = serverTransport.Accept(); + if (client != null) + { + inputTransport = inputTransportFactory.GetTransport(client); + outputTransport = outputTransportFactory.GetTransport(client); + inputProtocol = inputProtocolFactory.GetProtocol(inputTransport); + outputProtocol = outputProtocolFactory.GetProtocol(outputTransport); + while (processor.Process(inputProtocol, outputProtocol)) { } + } + } + catch (TTransportException ttx) + { + // Client died, just move on + if (stop) + { + logDelegate("TSimpleServer was shutting down, caught " + ttx.GetType().Name); + } + } + catch (Exception x) + { + logDelegate(x.ToString()); + } + + if (inputTransport != null) + { + inputTransport.Close(); + } + + if (outputTransport != null) + { + outputTransport.Close(); + } + } + + if (stop) + { + try + { + serverTransport.Close(); + } + catch (TTransportException ttx) + { + logDelegate("TServerTranport failed on close: " + ttx.Message); + } + stop = false; + } + } + + public override void Stop() + { + stop = true; + serverTransport.Close(); + } + } +} diff --git a/lib/csharp/src/Server/TThreadPoolServer.cs b/lib/csharp/src/Server/TThreadPoolServer.cs new file mode 100644 index 00000000..efc71f01 --- /dev/null +++ b/lib/csharp/src/Server/TThreadPoolServer.cs @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Threading; +using Thrift.Protocol; +using Thrift.Transport; + +namespace Thrift.Server +{ + /// + /// Server that uses C# built-in ThreadPool to spawn threads when handling requests + /// + public class TThreadPoolServer : TServer + { + private const int DEFAULT_MIN_THREADS = 10; + private const int DEFAULT_MAX_THREADS = 100; + private volatile bool stop = false; + + public TThreadPoolServer(TProcessor processor, TServerTransport serverTransport) + :this(processor, serverTransport, + new TTransportFactory(), new TTransportFactory(), + new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory(), + DEFAULT_MIN_THREADS, DEFAULT_MAX_THREADS, DefaultLogDelegate) + { + } + + public TThreadPoolServer(TProcessor processor, TServerTransport serverTransport, LogDelegate logDelegate) + : this(processor, serverTransport, + new TTransportFactory(), new TTransportFactory(), + new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory(), + DEFAULT_MIN_THREADS, DEFAULT_MAX_THREADS, logDelegate) + { + } + + + public TThreadPoolServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory transportFactory, + TProtocolFactory protocolFactory) + :this(processor, serverTransport, + transportFactory, transportFactory, + protocolFactory, protocolFactory, + DEFAULT_MIN_THREADS, DEFAULT_MAX_THREADS, DefaultLogDelegate) + { + } + + public TThreadPoolServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory inputTransportFactory, + TTransportFactory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory, + int minThreadPoolThreads, int maxThreadPoolThreads, LogDelegate logDel) + :base(processor, serverTransport, inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory, logDel) + { + if (!ThreadPool.SetMinThreads(minThreadPoolThreads, minThreadPoolThreads)) + { + throw new Exception("Error: could not SetMinThreads in ThreadPool"); + } + if (!ThreadPool.SetMaxThreads(maxThreadPoolThreads, maxThreadPoolThreads)) + { + throw new Exception("Error: could not SetMaxThreads in ThreadPool"); + } + } + + /// + /// Use new ThreadPool thread for each new client connection + /// + public override void Serve() + { + try + { + serverTransport.Listen(); + } + catch (TTransportException ttx) + { + logDelegate("Error, could not listen on ServerTransport: " + ttx); + return; + } + + while (!stop) + { + int failureCount = 0; + try + { + TTransport client = serverTransport.Accept(); + ThreadPool.QueueUserWorkItem(this.Execute, client); + } + catch (TTransportException ttx) + { + if (stop) + { + logDelegate("TThreadPoolServer was shutting down, caught " + ttx.GetType().Name); + } + else + { + ++failureCount; + logDelegate(ttx.ToString()); + } + + } + } + + if (stop) + { + try + { + serverTransport.Close(); + } + catch (TTransportException ttx) + { + logDelegate("TServerTransport failed on close: " + ttx.Message); + } + stop = false; + } + } + + /// + /// Loops on processing a client forever + /// threadContext will be a TTransport instance + /// + /// + private void Execute(Object threadContext) + { + TTransport client = (TTransport)threadContext; + TTransport inputTransport = null; + TTransport outputTransport = null; + TProtocol inputProtocol = null; + TProtocol outputProtocol = null; + try + { + inputTransport = inputTransportFactory.GetTransport(client); + outputTransport = outputTransportFactory.GetTransport(client); + inputProtocol = inputProtocolFactory.GetProtocol(inputTransport); + outputProtocol = outputProtocolFactory.GetProtocol(outputTransport); + while (processor.Process(inputProtocol, outputProtocol)) + { + //keep processing requests until client disconnects + } + } + catch (TTransportException) + { + // Assume the client died and continue silently + //Console.WriteLine(ttx); + } + + catch (Exception x) + { + logDelegate("Error: " + x); + } + + if (inputTransport != null) + { + inputTransport.Close(); + } + if (outputTransport != null) + { + outputTransport.Close(); + } + } + + public override void Stop() + { + stop = true; + serverTransport.Close(); + } + } +} diff --git a/lib/csharp/src/Server/TThreadedServer.cs b/lib/csharp/src/Server/TThreadedServer.cs new file mode 100644 index 00000000..75206f15 --- /dev/null +++ b/lib/csharp/src/Server/TThreadedServer.cs @@ -0,0 +1,234 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using Thrift.Collections; +using Thrift.Protocol; +using Thrift.Transport; + +namespace Thrift.Server +{ + /// + /// Server that uses C# threads (as opposed to the ThreadPool) when handling requests + /// + public class TThreadedServer : TServer + { + private const int DEFAULT_MAX_THREADS = 100; + private volatile bool stop = false; + private readonly int maxThreads; + + private Queue clientQueue; + private THashSet clientThreads; + private object clientLock; + private Thread workerThread; + + public TThreadedServer(TProcessor processor, TServerTransport serverTransport) + : this(processor, serverTransport, + new TTransportFactory(), new TTransportFactory(), + new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory(), + DEFAULT_MAX_THREADS, DefaultLogDelegate) + { + } + + public TThreadedServer(TProcessor processor, TServerTransport serverTransport, LogDelegate logDelegate) + : this(processor, serverTransport, + new TTransportFactory(), new TTransportFactory(), + new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory(), + DEFAULT_MAX_THREADS, logDelegate) + { + } + + + public TThreadedServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory transportFactory, + TProtocolFactory protocolFactory) + : this(processor, serverTransport, + transportFactory, transportFactory, + protocolFactory, protocolFactory, + DEFAULT_MAX_THREADS, DefaultLogDelegate) + { + } + + public TThreadedServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory inputTransportFactory, + TTransportFactory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory, + int maxThreads, LogDelegate logDel) + : base(processor, serverTransport, inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory, logDel) + { + this.maxThreads = maxThreads; + clientQueue = new Queue(); + clientLock = new object(); + clientThreads = new THashSet(); + } + + /// + /// Use new Thread for each new client connection. block until numConnections < maxTHreads + /// + public override void Serve() + { + try + { + //start worker thread + workerThread = new Thread(new ThreadStart(Execute)); + workerThread.Start(); + serverTransport.Listen(); + } + catch (TTransportException ttx) + { + logDelegate("Error, could not listen on ServerTransport: " + ttx); + return; + } + + while (!stop) + { + int failureCount = 0; + try + { + TTransport client = serverTransport.Accept(); + lock (clientLock) + { + clientQueue.Enqueue(client); + Monitor.Pulse(clientLock); + } + } + catch (TTransportException ttx) + { + if (stop) + { + logDelegate("TThreadPoolServer was shutting down, caught " + ttx); + } + else + { + ++failureCount; + logDelegate(ttx.ToString()); + } + + } + } + + if (stop) + { + try + { + serverTransport.Close(); + } + catch (TTransportException ttx) + { + logDelegate("TServeTransport failed on close: " + ttx.Message); + } + stop = false; + } + } + + /// + /// Loops on processing a client forever + /// threadContext will be a TTransport instance + /// + /// + private void Execute() + { + while (!stop) + { + TTransport client; + Thread t; + lock (clientLock) + { + //don't dequeue if too many connections + while (clientThreads.Count >= maxThreads) + { + Monitor.Wait(clientLock); + } + + while (clientQueue.Count == 0) + { + Monitor.Wait(clientLock); + } + + client = clientQueue.Dequeue(); + t = new Thread(new ParameterizedThreadStart(ClientWorker)); + clientThreads.Add(t); + } + //start processing requests from client on new thread + t.Start(client); + } + } + + private void ClientWorker(Object context) + { + TTransport client = (TTransport)context; + TTransport inputTransport = null; + TTransport outputTransport = null; + TProtocol inputProtocol = null; + TProtocol outputProtocol = null; + try + { + inputTransport = inputTransportFactory.GetTransport(client); + outputTransport = outputTransportFactory.GetTransport(client); + inputProtocol = inputProtocolFactory.GetProtocol(inputTransport); + outputProtocol = outputProtocolFactory.GetProtocol(outputTransport); + while (processor.Process(inputProtocol, outputProtocol)) + { + //keep processing requests until client disconnects + } + } + catch (TTransportException) + { + } + catch (Exception x) + { + logDelegate("Error: " + x); + } + + if (inputTransport != null) + { + inputTransport.Close(); + } + if (outputTransport != null) + { + outputTransport.Close(); + } + + lock (clientLock) + { + clientThreads.Remove(Thread.CurrentThread); + Monitor.Pulse(clientLock); + } + return; + } + + public override void Stop() + { + stop = true; + serverTransport.Close(); + //clean up all the threads myself + workerThread.Abort(); + foreach (Thread t in clientThreads) + { + t.Abort(); + } + } + } +} diff --git a/lib/csharp/src/TApplicationException.cs b/lib/csharp/src/TApplicationException.cs new file mode 100644 index 00000000..12719686 --- /dev/null +++ b/lib/csharp/src/TApplicationException.cs @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using Thrift.Protocol; + +namespace Thrift +{ + public class TApplicationException : Exception + { + protected ExceptionType type; + + public TApplicationException() + { + } + + public TApplicationException(ExceptionType type) + { + this.type = type; + } + + public TApplicationException(ExceptionType type, string message) + : base(message) + { + this.type = type; + } + + public static TApplicationException Read(TProtocol iprot) + { + TField field; + + string message = null; + ExceptionType type = ExceptionType.Unknown; + + while (true) + { + field = iprot.ReadFieldBegin(); + if (field.Type == TType.Stop) + { + break; + } + + switch (field.ID) + { + case 1: + if (field.Type == TType.String) + { + message = iprot.ReadString(); + } + else + { + TProtocolUtil.Skip(iprot, field.Type); + } + break; + case 2: + if (field.Type == TType.I32) + { + type = (ExceptionType)iprot.ReadI32(); + } + else + { + TProtocolUtil.Skip(iprot, field.Type); + } + break; + default: + TProtocolUtil.Skip(iprot, field.Type); + break; + } + + iprot.ReadFieldEnd(); + } + + iprot.ReadStructEnd(); + + return new TApplicationException(type, message); + } + + public void Write(TProtocol oprot) + { + TStruct struc = new TStruct("TApplicationException"); + TField field = new TField(); + + oprot.WriteStructBegin(struc); + + if (!String.IsNullOrEmpty(Message)) + { + field.Name = "message"; + field.Type = TType.String; + field.ID = 1; + oprot.WriteFieldBegin(field); + oprot.WriteString(Message); + oprot.WriteFieldEnd(); + } + + field.Name = "type"; + field.Type = TType.I32; + field.ID = 2; + oprot.WriteFieldBegin(field); + oprot.WriteI32((int)type); + oprot.WriteFieldEnd(); + oprot.WriteFieldStop(); + oprot.WriteStructEnd(); + } + + public enum ExceptionType + { + Unknown, + UnknownMethod, + InvalidMessageType, + WrongMethodName, + BadSequenceID, + MissingResult + } + } +} diff --git a/lib/csharp/src/TProcessor.cs b/lib/csharp/src/TProcessor.cs new file mode 100644 index 00000000..cbb55b79 --- /dev/null +++ b/lib/csharp/src/TProcessor.cs @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using Thrift.Protocol; + +namespace Thrift +{ + public interface TProcessor + { + bool Process(TProtocol iprot, TProtocol oprot); + } +} diff --git a/lib/csharp/src/Thrift.csproj b/lib/csharp/src/Thrift.csproj new file mode 100644 index 00000000..1eb4355d --- /dev/null +++ b/lib/csharp/src/Thrift.csproj @@ -0,0 +1,77 @@ + + + Debug + AnyCPU + {499EB63C-D74C-47E8-AE48-A2FC94538E9D} + 9.0.21022 + 2.0 + Library + false + Thrift + v3.5 + 512 + Thrift + SAK + SAK + SAK + SAK + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + 3.5 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lib/csharp/src/Thrift.sln b/lib/csharp/src/Thrift.sln new file mode 100644 index 00000000..cb7342cd --- /dev/null +++ b/lib/csharp/src/Thrift.sln @@ -0,0 +1,51 @@ + +Microsoft Visual Studio Solution File, Format Version 10.00 +# Visual Studio 2008 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Thrift", "Thrift.csproj", "{499EB63C-D74C-47E8-AE48-A2FC94538E9D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ThriftTest", "..\..\..\test\csharp\ThriftTest\ThriftTest.csproj", "{48DD757F-CA95-4DD7-BDA4-58DB6F108C2C}" + ProjectSection(ProjectDependencies) = postProject + {499EB63C-D74C-47E8-AE48-A2FC94538E9D} = {499EB63C-D74C-47E8-AE48-A2FC94538E9D} + EndProjectSection +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ThriftMSBuildTask", "..\ThriftMSBuildTask\ThriftMSBuildTask.csproj", "{EC0A0231-66EA-4593-A792-C6CA3BB8668E}" +EndProject +Global + GlobalSection(SourceCodeControl) = preSolution + SccNumberOfProjects = 4 + SccProjectName0 = Perforce\u0020Project + SccLocalPath0 = ..\\..\\.. + SccProvider0 = MSSCCI:Perforce\u0020SCM + SccProjectFilePathRelativizedFromConnection0 = lib\\csharp\\src\\ + SccProjectUniqueName1 = Thrift.csproj + SccLocalPath1 = ..\\..\\.. + SccProjectFilePathRelativizedFromConnection1 = lib\\csharp\\src\\ + SccProjectUniqueName2 = ..\\..\\..\\test\\csharp\\ThriftTest\\ThriftTest.csproj + SccLocalPath2 = ..\\..\\.. + SccProjectFilePathRelativizedFromConnection2 = test\\csharp\\ThriftTest\\ + SccProjectUniqueName3 = ..\\ThriftMSBuildTask\\ThriftMSBuildTask.csproj + SccLocalPath3 = ..\\..\\.. + SccProjectFilePathRelativizedFromConnection3 = lib\\csharp\\ThriftMSBuildTask\\ + EndGlobalSection + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {499EB63C-D74C-47E8-AE48-A2FC94538E9D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {499EB63C-D74C-47E8-AE48-A2FC94538E9D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {499EB63C-D74C-47E8-AE48-A2FC94538E9D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {499EB63C-D74C-47E8-AE48-A2FC94538E9D}.Release|Any CPU.Build.0 = Release|Any CPU + {48DD757F-CA95-4DD7-BDA4-58DB6F108C2C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {48DD757F-CA95-4DD7-BDA4-58DB6F108C2C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {48DD757F-CA95-4DD7-BDA4-58DB6F108C2C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {48DD757F-CA95-4DD7-BDA4-58DB6F108C2C}.Release|Any CPU.Build.0 = Release|Any CPU + {EC0A0231-66EA-4593-A792-C6CA3BB8668E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EC0A0231-66EA-4593-A792-C6CA3BB8668E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EC0A0231-66EA-4593-A792-C6CA3BB8668E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EC0A0231-66EA-4593-A792-C6CA3BB8668E}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/lib/csharp/src/Transport/TBufferedTransport.cs b/lib/csharp/src/Transport/TBufferedTransport.cs new file mode 100644 index 00000000..28a855a5 --- /dev/null +++ b/lib/csharp/src/Transport/TBufferedTransport.cs @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.IO; + +namespace Thrift.Transport +{ + public class TBufferedTransport : TTransport + { + private BufferedStream inputBuffer; + private BufferedStream outputBuffer; + private int bufSize; + private TStreamTransport transport; + + public TBufferedTransport(TStreamTransport transport) + :this(transport, 1024) + { + + } + + public TBufferedTransport(TStreamTransport transport, int bufSize) + { + this.bufSize = bufSize; + this.transport = transport; + InitBuffers(); + } + + private void InitBuffers() + { + if (transport.InputStream != null) + { + inputBuffer = new BufferedStream(transport.InputStream, bufSize); + } + if (transport.OutputStream != null) + { + outputBuffer = new BufferedStream(transport.OutputStream, bufSize); + } + } + + public TTransport UnderlyingTransport + { + get { return transport; } + } + + public override bool IsOpen + { + get { return transport.IsOpen; } + } + + public override void Open() + { + transport.Open(); + InitBuffers(); + } + + public override void Close() + { + if (inputBuffer != null && inputBuffer.CanRead) + { + inputBuffer.Close(); + } + if (outputBuffer != null && outputBuffer.CanWrite) + { + outputBuffer.Close(); + } + } + + public override int Read(byte[] buf, int off, int len) + { + return inputBuffer.Read(buf, off, len); + } + + public override void Write(byte[] buf, int off, int len) + { + outputBuffer.Write(buf, off, len); + } + + public override void Flush() + { + outputBuffer.Flush(); + } + } +} diff --git a/lib/csharp/src/Transport/TServerSocket.cs b/lib/csharp/src/Transport/TServerSocket.cs new file mode 100644 index 00000000..2658fce8 --- /dev/null +++ b/lib/csharp/src/Transport/TServerSocket.cs @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Net.Sockets; + + +namespace Thrift.Transport +{ + public class TServerSocket : TServerTransport + { + /** + * Underlying server with socket + */ + private TcpListener server = null; + + /** + * Port to listen on + */ + private int port = 0; + + /** + * Timeout for client sockets from accept + */ + private int clientTimeout = 0; + + /** + * Whether or not to wrap new TSocket connections in buffers + */ + private bool useBufferedSockets = false; + + /** + * Creates a server socket from underlying socket object + */ + public TServerSocket(TcpListener listener) + :this(listener, 0) + { + } + + /** + * Creates a server socket from underlying socket object + */ + public TServerSocket(TcpListener listener, int clientTimeout) + { + this.server = listener; + this.clientTimeout = clientTimeout; + } + + /** + * Creates just a port listening server socket + */ + public TServerSocket(int port) + : this(port, 0) + { + } + + /** + * Creates just a port listening server socket + */ + public TServerSocket(int port, int clientTimeout) + :this(port, clientTimeout, false) + { + } + + public TServerSocket(int port, int clientTimeout, bool useBufferedSockets) + { + this.port = port; + this.clientTimeout = clientTimeout; + this.useBufferedSockets = useBufferedSockets; + try + { + // Make server socket + server = new TcpListener(System.Net.IPAddress.Any, this.port); + } + catch (Exception) + { + server = null; + throw new TTransportException("Could not create ServerSocket on port " + port + "."); + } + } + + public override void Listen() + { + // Make sure not to block on accept + if (server != null) + { + try + { + server.Start(); + } + catch (SocketException sx) + { + throw new TTransportException("Could not accept on listening socket: " + sx.Message); + } + } + } + + protected override TTransport AcceptImpl() + { + if (server == null) + { + throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket."); + } + try + { + TcpClient result = server.AcceptTcpClient(); + TSocket result2 = new TSocket(result); + result2.Timeout = clientTimeout; + if (useBufferedSockets) + { + TBufferedTransport result3 = new TBufferedTransport(result2); + return result3; + } + else + { + return result2; + } + } + catch (Exception ex) + { + throw new TTransportException(ex.ToString()); + } + } + + public override void Close() + { + if (server != null) + { + try + { + server.Stop(); + } + catch (Exception ex) + { + throw new TTransportException("WARNING: Could not close server socket: " + ex); + } + server = null; + } + } + } +} diff --git a/lib/csharp/src/Transport/TServerTransport.cs b/lib/csharp/src/Transport/TServerTransport.cs new file mode 100644 index 00000000..9cb52e5c --- /dev/null +++ b/lib/csharp/src/Transport/TServerTransport.cs @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; + +namespace Thrift.Transport +{ + public abstract class TServerTransport + { + public abstract void Listen(); + public abstract void Close(); + protected abstract TTransport AcceptImpl(); + + public TTransport Accept() + { + TTransport transport = AcceptImpl(); + if (transport == null) { + throw new TTransportException("accept() may not return NULL"); + } + return transport; + } + } +} diff --git a/lib/csharp/src/Transport/TSocket.cs b/lib/csharp/src/Transport/TSocket.cs new file mode 100644 index 00000000..18cf1547 --- /dev/null +++ b/lib/csharp/src/Transport/TSocket.cs @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Net.Sockets; + +namespace Thrift.Transport +{ + public class TSocket : TStreamTransport + { + private TcpClient client = null; + private string host = null; + private int port = 0; + private int timeout = 0; + + public TSocket(TcpClient client) + { + this.client = client; + + if (IsOpen) + { + inputStream = client.GetStream(); + outputStream = client.GetStream(); + } + } + + public TSocket(string host, int port) : this(host, port, 0) + { + } + + public TSocket(string host, int port, int timeout) + { + this.host = host; + this.port = port; + this.timeout = timeout; + + InitSocket(); + } + + private void InitSocket() + { + client = new TcpClient(); + client.ReceiveTimeout = client.SendTimeout = timeout; + } + + public int Timeout + { + set + { + client.ReceiveTimeout = client.SendTimeout = timeout = value; + } + } + + public TcpClient TcpClient + { + get + { + return client; + } + } + + public string Host + { + get + { + return host; + } + } + + public int Port + { + get + { + return port; + } + } + + public override bool IsOpen + { + get + { + if (client == null) + { + return false; + } + + return client.Connected; + } + } + + public override void Open() + { + if (IsOpen) + { + throw new TTransportException(TTransportException.ExceptionType.AlreadyOpen, "Socket already connected"); + } + + if (String.IsNullOrEmpty(host)) + { + throw new TTransportException(TTransportException.ExceptionType.NotOpen, "Cannot open null host"); + } + + if (port <= 0) + { + throw new TTransportException(TTransportException.ExceptionType.NotOpen, "Cannot open without port"); + } + + if (client == null) + { + InitSocket(); + } + + client.Connect(host, port); + inputStream = client.GetStream(); + outputStream = client.GetStream(); + } + + public override void Close() + { + base.Close(); + if (client != null) + { + client.Close(); + client = null; + } + } + } +} diff --git a/lib/csharp/src/Transport/TStreamTransport.cs b/lib/csharp/src/Transport/TStreamTransport.cs new file mode 100644 index 00000000..7681e0d9 --- /dev/null +++ b/lib/csharp/src/Transport/TStreamTransport.cs @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.IO; + +namespace Thrift.Transport +{ + public class TStreamTransport : TTransport + { + protected Stream inputStream; + protected Stream outputStream; + + public TStreamTransport() + { + } + + public TStreamTransport(Stream inputStream, Stream outputStream) + { + this.inputStream = inputStream; + this.outputStream = outputStream; + } + + public Stream OutputStream + { + get { return outputStream; } + } + + public Stream InputStream + { + get { return inputStream; } + } + + public override bool IsOpen + { + get { return true; } + } + + public override void Open() + { + } + + public override void Close() + { + if (inputStream != null) + { + inputStream.Close(); + inputStream = null; + } + if (outputStream != null) + { + outputStream.Close(); + outputStream = null; + } + } + + public override int Read(byte[] buf, int off, int len) + { + if (inputStream == null) + { + throw new TTransportException(TTransportException.ExceptionType.NotOpen, "Cannot read from null inputstream"); + } + + return inputStream.Read(buf, off, len); + } + + public override void Write(byte[] buf, int off, int len) + { + if (outputStream == null) + { + throw new TTransportException(TTransportException.ExceptionType.NotOpen, "Cannot write to null outputstream"); + } + + outputStream.Write(buf, off, len); + } + + public override void Flush() + { + if (outputStream == null) + { + throw new TTransportException(TTransportException.ExceptionType.NotOpen, "Cannot flush null outputstream"); + } + + outputStream.Flush(); + } + } +} diff --git a/lib/csharp/src/Transport/TTransport.cs b/lib/csharp/src/Transport/TTransport.cs new file mode 100644 index 00000000..83f6776c --- /dev/null +++ b/lib/csharp/src/Transport/TTransport.cs @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; + +namespace Thrift.Transport +{ + public abstract class TTransport + { + public abstract bool IsOpen + { + get; + } + + public bool Peek() + { + return IsOpen; + } + + public abstract void Open(); + + public abstract void Close(); + + public abstract int Read(byte[] buf, int off, int len); + + public int ReadAll(byte[] buf, int off, int len) + { + int got = 0; + int ret = 0; + + while (got < len) + { + ret = Read(buf, off + got, len - got); + if (ret <= 0) + { + throw new TTransportException("Cannot read, Remote side has closed"); + } + got += ret; + } + + return got; + } + + public abstract void Write(byte[] buf, int off, int len); + + public virtual void Flush() + { + } + } +} diff --git a/lib/csharp/src/Transport/TTransportException.cs b/lib/csharp/src/Transport/TTransportException.cs new file mode 100644 index 00000000..fe10faa5 --- /dev/null +++ b/lib/csharp/src/Transport/TTransportException.cs @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; + +namespace Thrift.Transport +{ + public class TTransportException : Exception + { + protected ExceptionType type; + + public TTransportException() + : base() + { + } + + public TTransportException(ExceptionType type) + : this() + { + this.type = type; + } + + public TTransportException(ExceptionType type, string message) + : base(message) + { + this.type = type; + } + + public TTransportException(string message) + : base(message) + { + } + + public ExceptionType Type + { + get { return type; } + } + + public enum ExceptionType + { + Unknown, + NotOpen, + AlreadyOpen, + TimedOut, + EndOfFile + } + } +} diff --git a/lib/csharp/src/Transport/TTransportFactory.cs b/lib/csharp/src/Transport/TTransportFactory.cs new file mode 100644 index 00000000..3d3694db --- /dev/null +++ b/lib/csharp/src/Transport/TTransportFactory.cs @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; + +namespace Thrift.Transport +{ + /// + /// From Mark Slee & Aditya Agarwal of Facebook: + /// Factory class used to create wrapped instance of Transports. + /// This is used primarily in servers, which get Transports from + /// a ServerTransport and then may want to mutate them (i.e. create + /// a BufferedTransport from the underlying base transport) + /// + public class TTransportFactory + { + public virtual TTransport GetTransport(TTransport trans) + { + return trans; + } + } +} diff --git a/lib/erl/Makefile b/lib/erl/Makefile new file mode 100644 index 00000000..77fe8b6a --- /dev/null +++ b/lib/erl/Makefile @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +MODULES = \ + src + +all clean docs: + for dir in $(MODULES); do \ + (cd $$dir; ${MAKE} $@); \ + done + +install: all + echo 'No install target, sorry.' + +check: all + +distclean: clean + +# Hack to make "make dist" work. +# This should not work, but it appears to. +distdir: diff --git a/lib/erl/README b/lib/erl/README new file mode 100644 index 00000000..ddb6946f --- /dev/null +++ b/lib/erl/README @@ -0,0 +1,56 @@ +Thrift Erlang Software Library + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Example +======= + +Example session using thrift_client: + +118> f(), {ok, C} = thrift_client:start_link("localhost", 9090, thriftTest_thrif +t). +{ok,<0.271.0>} +119> thrift_client:call(C, testVoid, []). +{ok,ok} +120> thrift_client:call(C, testVoid, [asdf]). +{error,{bad_args,testVoid,[asdf]}} +121> thrift_client:call(C, testI32, [123]). +{ok,123} +122> thrift_client:call(C, testOneway, [1]). +{ok,ok} +123> catch thrift_client:call(C, testXception, ["foo"]). +{error,{no_function,testXception}} +124> catch thrift_client:call(C, testException, ["foo"]). +{ok,ok} +125> catch thrift_client:call(C, testException, ["Xception"]). +{xception,1001,"This is an Xception"} +126> thrift_client:call(C, testException, ["Xception"]). + +=ERROR REPORT==== 24-Feb-2008::23:00:23 === +Error in process <0.269.0> with exit value: {{nocatch,{xception,1001,"This is an + Xception"}},[{thrift_client,call,3},{erl_eval,do_apply,5},{shell,exprs,6},{shel +l,eval_loop,3}]} + +** exited: {{nocatch,{xception,1001,"This is an Xception"}}, + [{thrift_client,call,3}, + {erl_eval,do_apply,5}, + {shell,exprs,6}, + {shell,eval_loop,3}]} ** diff --git a/lib/erl/build/beamver b/lib/erl/build/beamver new file mode 100644 index 00000000..2b5f77b3 --- /dev/null +++ b/lib/erl/build/beamver @@ -0,0 +1,59 @@ +#!/bin/sh + +# erlwareSys: otp/build/beamver,v 1.1 2002/02/14 11:45:20 hal Exp $ + +# usage: beamver +# +# if there's a usable -vsn() attribute, print it and exit with status 0 +# otherwise, print nothing and exit with status 1 + +# From the Erlang shell: +# +# 5> code:which(acca_inets). +# "/home/martin/work/otp/releases//../../acca/ebin/.beam" +# +# 8> beam_lib:version(code:which()). +# {ok,{,['$Id: beamver,v 1.1.1.1 2003/06/13 21:43:21 mlogan Exp $ ']}} + +# TMPFILE looks like this: +# +# io:format("hello ~p~n", +# beam_lib:version("/home/hal/work/otp/acca/ebin/acca_inets.beam")]). + +TMPFILE=/tmp/beamver.$$ + +# exit with failure if we can't read the file +test -f "$1" || exit 1 +BEAMFILE=\"$1\" + +cat > $TMPFILE <<_EOF +io:format("~p~n", + [beam_lib:version($BEAMFILE)]). +_EOF + +# beam_result is {ok,{Module_name, Beam_version} or {error,beam_lib,{Reason}} +beam_result=`erl -noshell \ + -s file eval $TMPFILE \ + -s erlang halt` + +rm -f $TMPFILE + +# sed regexes: +# remove brackets and anything outside them +# remove quotes and anything outside them +# remove apostrophes and anything outside them +# remove leading and trailing spaces + +case $beam_result in +\{ok*) + echo $beam_result | sed -e 's/.*\[\(.*\)].*/\1/' \ + -e 's/.*\"\(.*\)\".*/\1/' \ + -e "s/.*\'\(.*\)\'.*/\1/" \ + -e 's/ *$//' -e 's/^ *//' + exit 0 + ;; +*) + exit 1 + ;; +esac + diff --git a/lib/erl/build/buildtargets.mk b/lib/erl/build/buildtargets.mk new file mode 100644 index 00000000..db52b785 --- /dev/null +++ b/lib/erl/build/buildtargets.mk @@ -0,0 +1,15 @@ +EBIN ?= ../ebin +ESRC ?= . +EMULATOR = beam + +ERLC_WFLAGS = -W +ERLC = erlc $(ERLC_WFLAGS) $(ERLC_FLAGS) +ERL = erl -boot start_clean + +$(EBIN)/%.beam: $(ESRC)/%.erl + @echo " ERLC $<" + @$(ERLC) $(ERL_FLAGS) $(ERL_COMPILE_FLAGS) -o$(EBIN) $< + +.erl.beam: + $(ERLC) $(ERL_FLAGS) $(ERL_COMPILE_FLAGS) -o$(dir $@) $< + diff --git a/lib/erl/build/colors.mk b/lib/erl/build/colors.mk new file mode 100644 index 00000000..4d69c41d --- /dev/null +++ b/lib/erl/build/colors.mk @@ -0,0 +1,24 @@ +# Colors to assist visual inspection of make output. + +# Colors +LGRAY=$$'\e[0;37m' +DGRAY=$$'\e[1;30m' +LGREEN=$$'\e[1;32m' +LBLUE=$$'\e[1;34m' +LCYAN=$$'\e[1;36m' +LPURPLE=$$'\e[1;35m' +LRED=$$'\e[1;31m' +NO_COLOR=$$'\e[0m' +DEFAULT=$$'\e[0m' +BLACK=$$'\e[0;30m' +BLUE=$$'\e[0;34m' +GREEN=$$'\e[0;32m' +CYAN=$$'\e[0;36m' +RED=$$'\e[0;31m' +PURPLE=$$'\e[0;35m' +BROWN=$$'\e[0;33m' +YELLOW=$$'\e[1;33m' +WHITE=$$'\e[1;37m' + +BOLD=$$'\e[1;37m' +OFF=$$'\e[0m' diff --git a/lib/erl/build/docs.mk b/lib/erl/build/docs.mk new file mode 100644 index 00000000..b0b7377f --- /dev/null +++ b/lib/erl/build/docs.mk @@ -0,0 +1,12 @@ +EDOC_PATH=../../../tools/utilities + +#single place to include docs from. +docs: + @mkdir -p ../doc + @echo -n $${MY_BLUE:-$(BLUE)}; \ + $(EDOC_PATH)/edoc $(APP_NAME); \ + if [ $$? -eq 0 ]; then \ + echo $${MY_LRED:-$(LRED)}"$$d Doc Failed"; \ + fi; \ + echo -n $(OFF)$(NO_COLOR) + diff --git a/lib/erl/build/mime.types b/lib/erl/build/mime.types new file mode 100644 index 00000000..d6e3c0d0 --- /dev/null +++ b/lib/erl/build/mime.types @@ -0,0 +1,98 @@ + +application/activemessage +application/andrew-inset +application/applefile +application/atomicmail +application/dca-rft +application/dec-dx +application/mac-binhex40 hqx +application/mac-compactpro cpt +application/macwriteii +application/msword doc +application/news-message-id +application/news-transmission +application/octet-stream bin dms lha lzh exe class +application/oda oda +application/pdf pdf +application/postscript ai eps ps +application/powerpoint ppt +application/remote-printing +application/rtf rtf +application/slate +application/wita +application/wordperfect5.1 +application/x-bcpio bcpio +application/x-cdlink vcd +application/x-compress Z +application/x-cpio cpio +application/x-csh csh +application/x-director dcr dir dxr +application/x-dvi dvi +application/x-gtar gtar +application/x-gzip gz +application/x-hdf hdf +application/x-httpd-cgi cgi +application/x-koan skp skd skt skm +application/x-latex latex +application/x-mif mif +application/x-netcdf nc cdf +application/x-sh sh +application/x-shar shar +application/x-stuffit sit +application/x-sv4cpio sv4cpio +application/x-sv4crc sv4crc +application/x-tar tar +application/x-tcl tcl +application/x-tex tex +application/x-texinfo texinfo texi +application/x-troff t tr roff +application/x-troff-man man +application/x-troff-me me +application/x-troff-ms ms +application/x-ustar ustar +application/x-wais-source src +application/zip zip +audio/basic au snd +audio/mpeg mpga mp2 +audio/x-aiff aif aiff aifc +audio/x-pn-realaudio ram +audio/x-pn-realaudio-plugin rpm +audio/x-realaudio ra +audio/x-wav wav +chemical/x-pdb pdb xyz +image/gif gif +image/ief ief +image/jpeg jpeg jpg jpe +image/png png +image/tiff tiff tif +image/x-cmu-raster ras +image/x-portable-anymap pnm +image/x-portable-bitmap pbm +image/x-portable-graymap pgm +image/x-portable-pixmap ppm +image/x-rgb rgb +image/x-xbitmap xbm +image/x-xpixmap xpm +image/x-xwindowdump xwd +message/external-body +message/news +message/partial +message/rfc822 +multipart/alternative +multipart/appledouble +multipart/digest +multipart/mixed +multipart/parallel +text/html html htm +text/x-server-parsed-html shtml +text/plain txt +text/richtext rtx +text/tab-separated-values tsv +text/x-setext etx +text/x-sgml sgml sgm +video/mpeg mpeg mpg mpe +video/quicktime qt mov +video/x-msvideo avi +video/x-sgi-movie movie +x-conference/x-cooltalk ice +x-world/x-vrml wrl vrml diff --git a/lib/erl/build/otp.mk b/lib/erl/build/otp.mk new file mode 100644 index 00000000..1d16e2c8 --- /dev/null +++ b/lib/erl/build/otp.mk @@ -0,0 +1,146 @@ +# +----------------------------------------------------------------------+ +# $Id: otp.mk,v 1.4 2004/07/01 14:57:10 tfee Exp $ +# +----------------------------------------------------------------------+ + +# otp.mk +# - to be included in all OTP Makefiles +# installed to /usr/local/include/erlang/otp.mk + +# gmake looks in /usr/local/include - that's hard-coded +# users of this file will use +# include erlang/top.mk + +# most interface files will be installed to $ERL_RUN_TOP/app-vsn/include/*.hrl + +# group owner for library/include directories +ERLANGDEV_GROUP=erlangdev + +# ERL_TOP is root of Erlang source tree +# ERL_RUN_TOP is root of Erlang target tree (some Ericsson Makefiles use $ROOT) +# ERLANG_OTP is target root for Erlang code +# - see sasl/systools reference manual page; grep "TEST" + +# OS_TYPE is FreeBSD, NetBSD, OpenBSD, Linux, SCO_SV, SunOS. +OS_TYPE=${shell uname} + +# MHOST is the host where this Makefile runs. +MHOST=${shell hostname -s} +ERL_COMPILE_FLAGS+=-W0 + +# The location of the erlang runtime system. +ifndef ERL_RUN_TOP +ERL_RUN_TOP=/usr/local/lib/erlang +endif + + +# Edit to reflect local environment. +# ifeq (${OS_TYPE},Linux) +# ERL_RUN_TOP=/usr/local/lib/erlang +# Note* ERL_RUN_TOP can be determined by starting an +# erlang shell and typing code:root_dir(). +# ERL_TOP=a symbolic link to the actual source top, which changes from version to version +# Note* ERL_TOP is the directory where the erlang +# source files reside. Make sure to run ./configure there. +# TARGET=i686-pc-linux-gnu +# Note* Target can be found in $ERL_TOP/erts +# endif + +# See above for directions. +ifeq (${OS_TYPE},Linux) +ERL_TOP=/opt/OTP_SRC +TARGET=i686-pc-linux-gnu +endif + +ERLANG_OTP=/usr/local/erlang/otp +VAR_OTP=/var/otp + + +# Aliases for common binaries +# Note - CFLAGS is modified in erlang.conf + + +################################ +# SunOS +################################ +ifeq (${OS_TYPE},SunOS) + + CC=gcc + CXX=g++ + AR=/usr/ccs/bin/ar + ARFLAGS=-rv + CXXFLAGS+=${CFLAGS} -I/usr/include/g++ + LD=/usr/ccs/bin/ld + RANLIB=/usr/ccs/bin/ranlib + +CFLAGS+=-Wall -pedantic -ansi -O +CORE=*.core +endif + + +################################ +# FreeBSD +################################ +ifeq (${OS_TYPE},FreeBSD) + + ifdef LINUXBIN + COMPAT_LINUX=/compat/linux + CC=${COMPAT_LINUX}/usr/bin/gcc + CXX=${COMPAT_LINUX}/usr/bin/g++ + AR=${COMPAT_LINUX}/usr/bin/ar + ARFLAGS=-rv + CXXFLAGS+=-fhandle-exceptions ${CFLAGS} -I${COMPAT_LINUX}/usr/include/g++ + LD=${COMPAT_LINUX}/usr/bin/ld + RANLIB=${COMPAT_LINUX}/usr/bin/ranlib + BRANDELF=brandelf -t Linux + else + CC=gcc + CXX=g++ + AR=/usr/bin/ar + ARFLAGS=-rv + CXXFLAGS+=-fhandle-exceptions ${CFLAGS} -I/usr/include/g++ + LD=/usr/bin/ld + RANLIB=/usr/bin/ranlib + BRANDELF=@true + + ifdef USES_PTHREADS + CFLAGS+=-D_THREAD_SAFE + LDFLAGS+=-lc_r + + # -pthread flag for 3.0+ + ifneq (${shell uname -r | cut -d. -f1},2) + CFLAGS+=-pthread + endif + endif + endif + +CFLAGS+=-Wall -pedantic -ansi -O -DFREEBSD +CORE=*.core +endif + +################################ +# OpenBSD +################################ +ifeq (${OS_TYPE},OpenBSD) + + CC=gcc + CXX=g++ + AR=/usr/bin/ar + ARFLAGS=-rv + CXXFLAGS+=${CFLAGS} -I/usr/include/g++ + LD=/usr/bin/ld + RANLIB=/usr/bin/ranlib + + ifdef USES_PTHREADS + CFLAGS+=-D_THREAD_SAFE + LDFLAGS+=-lc_r + + # -pthread flag for 3.0+ + ifneq (${shell uname -r | cut -d. -f1},2) + CFLAGS+=-pthread + endif + endif + +CFLAGS+=-Wall -pedantic -ansi -O -DOPENBSD +CORE=*.core +endif + diff --git a/lib/erl/build/otp_subdir.mk b/lib/erl/build/otp_subdir.mk new file mode 100644 index 00000000..2a36c658 --- /dev/null +++ b/lib/erl/build/otp_subdir.mk @@ -0,0 +1,85 @@ +# Comment by tfee 2004-07-01 +# ========================== +# This file is a mod of the stock OTP one. +# The change allows make to stop when a compile error occurs. +# This file needs to go into two places: +# /usr/local/include/erlang +# /opt/OTP_SRC/make +# +# where OTP_SRC is a symbolic link to a peer directory containing +# the otp source, e.g. otp_src_R9C-2. +# +# After installing OTP, running sudo make install in otp/build +# will push this file out to the two places listed above. +# +# The mod involves setting the shell variable $short_circuit, which we +# introduce - ie it is not in the stock file. This variable is tested +# to affect execution flow and is also returned to affect the flow in +# the calling script (this one). The latter step is necessary because +# of the recursion involved. +# ===================================================================== + + +# ``The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved via the world wide web at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# The Initial Developer of the Original Code is Ericsson Utvecklings AB. +# Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings +# AB. All Rights Reserved.'' +# +# $Id: otp_subdir.mk,v 1.5 2004/07/12 15:12:23 jeinhorn Exp $ +# +# +# Make include file for otp + +.PHONY: debug opt release docs release_docs tests release_tests \ + clean depend + +# +# Targets that don't affect documentation directories +# +debug opt release docs release_docs tests release_tests clean depend: prepare + @set -e ; \ + app_pwd=`pwd` ; \ + if test -f vsn.mk; then \ + echo "=== Entering application" `basename $$app_pwd` ; \ + fi ; \ + case "$(MAKE)" in *clearmake*) tflag="-T";; *) tflag="";; esac; \ + short_circuit=0 ; \ + for d in $(SUB_DIRECTORIES); do \ + if [[ $$short_circuit = 0 ]]; then \ + if test -f $$d/SKIP ; then \ + echo "=== Skipping subdir $$d, reason:" ; \ + cat $$d/SKIP ; \ + echo "===" ; \ + else \ + if test ! -d $$d ; then \ + echo "=== Skipping subdir $$d, it is missing" ; \ + else \ + xflag="" ; \ + if test -f $$d/ignore_config_record.inf; then \ + xflag=$$tflag ; \ + fi ; \ + (cd $$d && $(MAKE) $$xflag $@) ; \ + if [[ $$? != 0 ]]; then \ + short_circuit=1 ; \ + fi ; \ + fi ; \ + fi ; \ + fi ; \ + done ; \ + if test -f vsn.mk; then \ + echo "=== Leaving application" `basename $$app_pwd` ; \ + fi ; \ + exit $$short_circuit + +prepare: + echo diff --git a/lib/erl/build/raw_test.mk b/lib/erl/build/raw_test.mk new file mode 100644 index 00000000..bf8535d1 --- /dev/null +++ b/lib/erl/build/raw_test.mk @@ -0,0 +1,29 @@ +# for testing erlang files directly. The set up for a +# this type of test would be +# files to test reside in lib//src and the test files which are +# just plain erlang code reside in lib//test +# +# This color codes emitted while the tests run assume that you are using +# a white-on-black display schema. If not, e.g. if you use a white +# background, you will not be able to read the "WHITE" text. +# You can override this by supplying your own "white" color, +# which may in fact be black! You do this by defining an environment +# variable named "MY_WHITE" and setting it to $'\e[0;30m' (which is +# simply bash's way of specifying "Escape [ 0 ; 3 0 m"). +# Similarly, you can set your versions of the standard colors +# found in colors.mk. + +test: + @TEST_MODULES=`ls *_test.erl`; \ + trap "echo $(OFF)$(NO_COLOR); exit 1;" 1 2 3 6; \ + for d in $$TEST_MODULES; do \ + echo $${MY_GREEN:-$(GREEN)}"Testing File $$d" $${MY_WHITE:-$(WHITE)}; \ + echo -n $${MY_BLUE:-$(BLUE)}; \ + erl -name $(APP_NAME) $(TEST_LIBS) \ + -s `basename $$d .erl` all -s init stop -noshell; \ + if [ $$? -ne 0 ]; then \ + echo $${MY_LRED:-$(LRED)}"$$d Test Failed"; \ + fi; \ + echo -n $(OFF)$(NO_COLOR); \ + done + diff --git a/lib/erl/include/thrift_constants.hrl b/lib/erl/include/thrift_constants.hrl new file mode 100644 index 00000000..36eb49bf --- /dev/null +++ b/lib/erl/include/thrift_constants.hrl @@ -0,0 +1,54 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +%% TType +-define(tType_STOP, 0). +-define(tType_VOID, 1). +-define(tType_BOOL, 2). +-define(tType_BYTE, 3). +-define(tType_DOUBLE, 4). +-define(tType_I16, 6). +-define(tType_I32, 8). +-define(tType_I64, 10). +-define(tType_STRING, 11). +-define(tType_STRUCT, 12). +-define(tType_MAP, 13). +-define(tType_SET, 14). +-define(tType_LIST, 15). + +% TMessageType +-define(tMessageType_CALL, 1). +-define(tMessageType_REPLY, 2). +-define(tMessageType_EXCEPTION, 3). +-define(tMessageType_ONEWAY, 4). + +% TApplicationException +-define(TApplicationException_Structure, + {struct, [{1, string}, + {2, i32}]}). + +-record('TApplicationException', {message, type}). + +-define(TApplicationException_UNKNOWN, 0). +-define(TApplicationException_UNKNOWN_METHOD, 1). +-define(TApplicationException_INVALID_MESSAGE_TYPE, 2). +-define(TApplicationException_WRONG_METHOD_NAME, 3). +-define(TApplicationException_BAD_SEQUENCE_ID, 4). +-define(TApplicationException_MISSING_RESULT, 5). + diff --git a/lib/erl/include/thrift_protocol.hrl b/lib/erl/include/thrift_protocol.hrl new file mode 100644 index 00000000..f4e1901f --- /dev/null +++ b/lib/erl/include/thrift_protocol.hrl @@ -0,0 +1,31 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-ifndef(THRIFT_PROTOCOL_INCLUDED). +-define(THRIFT_PROTOCOL_INCLUDED, yea). + +-record(protocol_message_begin, {name, type, seqid}). +-record(protocol_struct_begin, {name}). +-record(protocol_field_begin, {name, type, id}). +-record(protocol_map_begin, {ktype, vtype, size}). +-record(protocol_list_begin, {etype, size}). +-record(protocol_set_begin, {etype, size}). + + +-endif. diff --git a/lib/erl/src/Makefile b/lib/erl/src/Makefile new file mode 100644 index 00000000..980af812 --- /dev/null +++ b/lib/erl/src/Makefile @@ -0,0 +1,116 @@ +# $Id: Makefile,v 1.3 2004/08/13 16:35:59 mlogan Exp $ +# +include ../build/otp.mk +include ../build/colors.mk +include ../build/buildtargets.mk + +# ---------------------------------------------------- +# Application version +# ---------------------------------------------------- + +include ../vsn.mk +APP_NAME=thrift +PFX=thrift +VSN=$(THRIFT_VSN) + +# ---------------------------------------------------- +# Install directory specification +# WARNING: INSTALL_DIR the command to install a directory. +# INSTALL_DST is the target directory +# ---------------------------------------------------- +INSTALL_DST = $(ERLANG_OTP)/lib/$(APP_NAME)-$(VSN) + +# ---------------------------------------------------- +# Target Specs +# ---------------------------------------------------- + + +MODULES = $(shell find . -name \*.erl | sed 's:^\./::' | sed 's/\.erl//') +MODULES_STRING_LIST = $(shell find . -name \*.erl | sed 's:^\./:":' | sed 's/\.erl/",/') + +HRL_FILES= +INTERNAL_HRL_FILES= $(APP_NAME).hrl +ERL_FILES= $(MODULES:%=%.erl) +DOC_FILES=$(ERL_FILES) + +APP_FILE= $(APP_NAME).app +APPUP_FILE= $(APP_NAME).appup + +APP_SRC= $(APP_FILE).src +APPUP_SRC= $(APPUP_FILE).src + +APP_TARGET= $(EBIN)/$(APP_FILE) +APPUP_TARGET= $(EBIN)/$(APPUP_FILE) + +BEAMS= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) +TARGET_FILES= $(BEAMS) $(APP_TARGET) $(APPUP_TARGET) + +WEB_TARGET=/var/yaws/www/$(APP_NAME) + +# ---------------------------------------------------- +# FLAGS +# ---------------------------------------------------- + +ERL_FLAGS += +ERL_INCLUDE = -I../include -I../../fslib/include -I../../system_status/include +ERL_COMPILE_FLAGS += $(ERL_INCLUDE) + +# ---------------------------------------------------- +# Targets +# ---------------------------------------------------- + +all debug opt: $(EBIN) $(TARGET_FILES) + +#$(EBIN)/rm_logger.beam: $(APP_NAME).hrl +include ../build/docs.mk + +# Note: In the open-source build clean must not destroy the preloaded +# beam files. +clean: + rm -f $(TARGET_FILES) + rm -f *~ + rm -f core + rm -rf $(EBIN) + rm -rf *html + +$(EBIN): + mkdir $(EBIN) + +dialyzer: $(TARGET_FILES) + dialyzer --src -r . $(ERL_INCLUDE) + +# ---------------------------------------------------- +# Special Build Targets +# ---------------------------------------------------- + +$(APP_TARGET): $(APP_SRC) ../vsn.mk $(BEAMS) + sed -e 's;%VSN%;$(VSN);' \ + -e 's;%PFX%;$(PFX);' \ + -e 's;%APP_NAME%;$(APP_NAME);' \ + -e 's;%MODULES%;%MODULES%$(MODULES_STRING_LIST);' \ + $< > $<".tmp" + sed -e 's/%MODULES%\(.*\),/\1/' \ + $<".tmp" > $@ + rm $<".tmp" + +$(APPUP_TARGET): $(APPUP_SRC) ../vsn.mk + sed -e 's;%VSN%;$(VSN);' $< > $@ + +$(WEB_TARGET): ../markup/* + rm -rf $(WEB_TARGET) + mkdir $(WEB_TARGET) + cp -r ../markup/ $(WEB_TARGET) + cp -r ../skins/ $(WEB_TARGET) + +# ---------------------------------------------------- +# Install Target +# ---------------------------------------------------- + +install: all $(WEB_TARGET) +# $(INSTALL_DIR) $(INSTALL_DST)/src +# $(INSTALL_DATA) $(ERL_FILES) $(INSTALL_DST)/src +# $(INSTALL_DATA) $(INTERNAL_HRL_FILES) $(INSTALL_DST)/src +# $(INSTALL_DIR) $(INSTALL_DST)/include +# $(INSTALL_DATA) $(HRL_FILES) $(INSTALL_DST)/include +# $(INSTALL_DIR) $(INSTALL_DST)/ebin +# $(INSTALL_DATA) $(TARGET_FILES) $(INSTALL_DST)/ebin diff --git a/lib/erl/src/test_handler.erl b/lib/erl/src/test_handler.erl new file mode 100644 index 00000000..28a3acd3 --- /dev/null +++ b/lib/erl/src/test_handler.erl @@ -0,0 +1,26 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(test_handler). + +-export([handle_function/2]). + +handle_function(add, Params = {A, B}) -> + io:format("Got params: ~p~n", [Params]), + {reply, A + B}. diff --git a/lib/erl/src/test_service.erl b/lib/erl/src/test_service.erl new file mode 100644 index 00000000..7aa4827f --- /dev/null +++ b/lib/erl/src/test_service.erl @@ -0,0 +1,29 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(test_service). +% +% Test service definition + +-export([function_info/2]). + +function_info(add, params_type) -> + {struct, [{1, i32}, + {2, i32}]}; +function_info(add, reply_type) -> i32. diff --git a/lib/erl/src/thrift.app.src b/lib/erl/src/thrift.app.src new file mode 100644 index 00000000..681b3eb3 --- /dev/null +++ b/lib/erl/src/thrift.app.src @@ -0,0 +1,44 @@ +%%% -*- mode:erlang -*- +{application, %APP_NAME%, + [ + % A quick description of the application. + {description, "Thrift bindings"}, + + % The version of the applicaton + {vsn, "%VSN%"}, + + % All modules used by the application. + {modules, [ + %MODULES% + ]}, + + % All of the registered names the application uses. This can be ignored. + {registered, []}, + + % Applications that are to be started prior to this one. This can be ignored + % leave it alone unless you understand it well and let the .rel files in + % your release handle this. + {applications, + [ + kernel, + stdlib + ]}, + + % OTP application loader will load, but not start, included apps. Again + % this can be ignored as well. To load but not start an application it + % is easier to include it in the .rel file followed by the atom 'none' + {included_applications, []}, + + % configuration parameters similar to those in the config file specified + % on the command line. can be fetched with gas:get_env + {env, [ + % If an error/crash occurs during processing of a function, + % should the TApplicationException serialized back to the client + % include the erlang backtrace? + {exceptions_include_traces, true} + ]}, + + % The Module and Args used to start this application. + {mod, {thrift_app, []}} + ] +}. diff --git a/lib/erl/src/thrift.appup.src b/lib/erl/src/thrift.appup.src new file mode 100644 index 00000000..54a63833 --- /dev/null +++ b/lib/erl/src/thrift.appup.src @@ -0,0 +1 @@ +{"%VSN%",[],[]}. diff --git a/lib/erl/src/thrift_base64_transport.erl b/lib/erl/src/thrift_base64_transport.erl new file mode 100644 index 00000000..9d13151c --- /dev/null +++ b/lib/erl/src/thrift_base64_transport.erl @@ -0,0 +1,64 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_base64_transport). + +-behaviour(thrift_transport). + +%% API +-export([new/1, new_transport_factory/1]). + +%% thrift_transport callbacks +-export([write/2, read/2, flush/1, close/1]). + +%% State +-record(b64_transport, {wrapped}). + +new(Wrapped) -> + State = #b64_transport{wrapped = Wrapped}, + thrift_transport:new(?MODULE, State). + + +write(#b64_transport{wrapped = Wrapped}, Data) -> + thrift_transport:write(Wrapped, base64:encode(iolist_to_binary(Data))). + + +%% base64 doesn't support reading quite yet since it would involve +%% nasty buffering and such +read(#b64_transport{wrapped = Wrapped}, Data) -> + {error, no_reads_allowed}. + + +flush(#b64_transport{wrapped = Wrapped}) -> + thrift_transport:write(Wrapped, <<"\n">>), + thrift_transport:flush(Wrapped). + + +close(Me = #b64_transport{wrapped = Wrapped}) -> + flush(Me), + thrift_transport:close(Wrapped). + + +%%%% FACTORY GENERATION %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +new_transport_factory(WrapFactory) -> + F = fun() -> + {ok, Wrapped} = WrapFactory(), + new(Wrapped) + end, + {ok, F}. diff --git a/lib/erl/src/thrift_binary_protocol.erl b/lib/erl/src/thrift_binary_protocol.erl new file mode 100644 index 00000000..ad533842 --- /dev/null +++ b/lib/erl/src/thrift_binary_protocol.erl @@ -0,0 +1,325 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_binary_protocol). + +-behavior(thrift_protocol). + +-include("thrift_constants.hrl"). +-include("thrift_protocol.hrl"). + +-export([new/1, new/2, + read/2, + write/2, + flush_transport/1, + close_transport/1, + + new_protocol_factory/2 + ]). + +-record(binary_protocol, {transport, + strict_read=true, + strict_write=true + }). + +-define(VERSION_MASK, 16#FFFF0000). +-define(VERSION_1, 16#80010000). +-define(TYPE_MASK, 16#000000ff). + +new(Transport) -> + new(Transport, _Options = []). + +new(Transport, Options) -> + State = #binary_protocol{transport = Transport}, + State1 = parse_options(Options, State), + thrift_protocol:new(?MODULE, State1). + +parse_options([], State) -> + State; +parse_options([{strict_read, Bool} | Rest], State) when is_boolean(Bool) -> + parse_options(Rest, State#binary_protocol{strict_read=Bool}); +parse_options([{strict_write, Bool} | Rest], State) when is_boolean(Bool) -> + parse_options(Rest, State#binary_protocol{strict_write=Bool}). + + +flush_transport(#binary_protocol{transport = Transport}) -> + thrift_transport:flush(Transport). + +close_transport(#binary_protocol{transport = Transport}) -> + thrift_transport:close(Transport). + +%%% +%%% instance methods +%%% + +write(This, #protocol_message_begin{ + name = Name, + type = Type, + seqid = Seqid}) -> + case This#binary_protocol.strict_write of + true -> + write(This, {i32, ?VERSION_1 bor Type}), + write(This, {string, Name}), + write(This, {i32, Seqid}); + false -> + write(This, {string, Name}), + write(This, {byte, Type}), + write(This, {i32, Seqid}) + end, + ok; + +write(This, message_end) -> ok; + +write(This, #protocol_field_begin{ + name = _Name, + type = Type, + id = Id}) -> + write(This, {byte, Type}), + write(This, {i16, Id}), + ok; + +write(This, field_stop) -> + write(This, {byte, ?tType_STOP}), + ok; + +write(This, field_end) -> ok; + +write(This, #protocol_map_begin{ + ktype = Ktype, + vtype = Vtype, + size = Size}) -> + write(This, {byte, Ktype}), + write(This, {byte, Vtype}), + write(This, {i32, Size}), + ok; + +write(This, map_end) -> ok; + +write(This, #protocol_list_begin{ + etype = Etype, + size = Size}) -> + write(This, {byte, Etype}), + write(This, {i32, Size}), + ok; + +write(This, list_end) -> ok; + +write(This, #protocol_set_begin{ + etype = Etype, + size = Size}) -> + write(This, {byte, Etype}), + write(This, {i32, Size}), + ok; + +write(This, set_end) -> ok; + +write(This, #protocol_struct_begin{}) -> ok; +write(This, struct_end) -> ok; + +write(This, {bool, true}) -> write(This, {byte, 1}); +write(This, {bool, false}) -> write(This, {byte, 0}); + +write(This, {byte, Byte}) -> + write(This, <>); + +write(This, {i16, I16}) -> + write(This, <>); + +write(This, {i32, I32}) -> + write(This, <>); + +write(This, {i64, I64}) -> + write(This, <>); + +write(This, {double, Double}) -> + write(This, <>); + +write(This, {string, Str}) when is_list(Str) -> + write(This, {i32, length(Str)}), + write(This, list_to_binary(Str)); + +write(This, {string, Bin}) when is_binary(Bin) -> + write(This, {i32, size(Bin)}), + write(This, Bin); + +%% Data :: iolist() +write(This, Data) -> + thrift_transport:write(This#binary_protocol.transport, Data). + +%% + +read(This, message_begin) -> + case read(This, ui32) of + {ok, Sz} when Sz band ?VERSION_MASK =:= ?VERSION_1 -> + %% we're at version 1 + {ok, Name} = read(This, string), + Type = Sz band ?TYPE_MASK, + {ok, SeqId} = read(This, i32), + #protocol_message_begin{name = binary_to_list(Name), + type = Type, + seqid = SeqId}; + + {ok, Sz} when Sz < 0 -> + %% there's a version number but it's unexpected + {error, {bad_binary_protocol_version, Sz}}; + + {ok, Sz} when This#binary_protocol.strict_read =:= true -> + %% strict_read is true and there's no version header; that's an error + {error, no_binary_protocol_version}; + + {ok, Sz} when This#binary_protocol.strict_read =:= false -> + %% strict_read is false, so just read the old way + {ok, Name} = read(This, Sz), + {ok, Type} = read(This, byte), + {ok, SeqId} = read(This, i32), + #protocol_message_begin{name = binary_to_list(Name), + type = Type, + seqid = SeqId}; + + Err = {error, closed} -> Err; + Err = {error, timeout}-> Err; + Err = {error, ebadf} -> Err + end; + +read(This, message_end) -> ok; + +read(This, struct_begin) -> ok; +read(This, struct_end) -> ok; + +read(This, field_begin) -> + case read(This, byte) of + {ok, Type = ?tType_STOP} -> + #protocol_field_begin{type = Type}; + {ok, Type} -> + {ok, Id} = read(This, i16), + #protocol_field_begin{type = Type, + id = Id} + end; + +read(This, field_end) -> ok; + +read(This, map_begin) -> + {ok, Ktype} = read(This, byte), + {ok, Vtype} = read(This, byte), + {ok, Size} = read(This, i32), + #protocol_map_begin{ktype = Ktype, + vtype = Vtype, + size = Size}; +read(This, map_end) -> ok; + +read(This, list_begin) -> + {ok, Etype} = read(This, byte), + {ok, Size} = read(This, i32), + #protocol_list_begin{etype = Etype, + size = Size}; +read(This, list_end) -> ok; + +read(This, set_begin) -> + {ok, Etype} = read(This, byte), + {ok, Size} = read(This, i32), + #protocol_set_begin{etype = Etype, + size = Size}; +read(This, set_end) -> ok; + +read(This, field_stop) -> + {ok, ?tType_STOP} = read(This, byte), + ok; + +%% + +read(This, bool) -> + case read(This, byte) of + {ok, Byte} -> {ok, Byte /= 0}; + Else -> Else + end; + +read(This, byte) -> + case read(This, 1) of + {ok, <>} -> {ok, Val}; + Else -> Else + end; + +read(This, i16) -> + case read(This, 2) of + {ok, <>} -> {ok, Val}; + Else -> Else + end; + +read(This, i32) -> + case read(This, 4) of + {ok, <>} -> {ok, Val}; + Else -> Else + end; + +%% unsigned ints aren't used by thrift itself, but it's used for the parsing +%% of the packet version header. Without this special function BEAM works fine +%% but hipe thinks it received a bad version header. +read(This, ui32) -> + case read(This, 4) of + {ok, <>} -> {ok, Val}; + Else -> Else + end; + +read(This, i64) -> + case read(This, 8) of + {ok, <>} -> {ok, Val}; + Else -> Else + end; + +read(This, double) -> + case read(This, 8) of + {ok, <>} -> {ok, Val}; + Else -> Else + end; + +% returns a binary directly, call binary_to_list if necessary +read(This, string) -> + {ok, Sz} = read(This, i32), + {ok, Bin} = read(This, Sz); + +read(This, 0) -> {ok, <<>>}; +read(This, Len) when is_integer(Len), Len >= 0 -> + thrift_transport:read(This#binary_protocol.transport, Len). + + +%%%% FACTORY GENERATION %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +-record(tbp_opts, {strict_read = true, + strict_write = true}). + +parse_factory_options([], Opts) -> + Opts; +parse_factory_options([{strict_read, Bool} | Rest], Opts) when is_boolean(Bool) -> + parse_factory_options(Rest, Opts#tbp_opts{strict_read=Bool}); +parse_factory_options([{strict_write, Bool} | Rest], Opts) when is_boolean(Bool) -> + parse_factory_options(Rest, Opts#tbp_opts{strict_write=Bool}). + + +%% returns a (fun() -> thrift_protocol()) +new_protocol_factory(TransportFactory, Options) -> + ParsedOpts = parse_factory_options(Options, #tbp_opts{}), + F = fun() -> + {ok, Transport} = TransportFactory(), + thrift_binary_protocol:new( + Transport, + [{strict_read, ParsedOpts#tbp_opts.strict_read}, + {strict_write, ParsedOpts#tbp_opts.strict_write}]) + end, + {ok, F}. + diff --git a/lib/erl/src/thrift_buffered_transport.erl b/lib/erl/src/thrift_buffered_transport.erl new file mode 100644 index 00000000..ebc16bd6 --- /dev/null +++ b/lib/erl/src/thrift_buffered_transport.erl @@ -0,0 +1,180 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_buffered_transport). + +-behaviour(gen_server). +-behaviour(thrift_transport). + +%% API +-export([new/1, new_transport_factory/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +%% thrift_transport callbacks +-export([write/2, read/2, flush/1, close/1]). + +-record(buffered_transport, {wrapped, % a thrift_transport + write_buffer % iolist() + }). + +%%==================================================================== +%% API +%%==================================================================== +%%-------------------------------------------------------------------- +%% Function: start_link() -> {ok,Pid} | ignore | {error,Error} +%% Description: Starts the server +%%-------------------------------------------------------------------- +new(WrappedTransport) -> + case gen_server:start_link(?MODULE, [WrappedTransport], []) of + {ok, Pid} -> + thrift_transport:new(?MODULE, Pid); + Else -> + Else + end. + + + +%%-------------------------------------------------------------------- +%% Function: write(Transport, Data) -> ok +%% +%% Data = iolist() +%% +%% Description: Writes data into the buffer +%%-------------------------------------------------------------------- +write(Transport, Data) -> + gen_server:call(Transport, {write, Data}). + +%%-------------------------------------------------------------------- +%% Function: flush(Transport) -> ok +%% +%% Description: Flushes the buffer through to the wrapped transport +%%-------------------------------------------------------------------- +flush(Transport) -> + gen_server:call(Transport, flush). + +%%-------------------------------------------------------------------- +%% Function: close(Transport) -> ok +%% +%% Description: Closes the transport and the wrapped transport +%%-------------------------------------------------------------------- +close(Transport) -> + gen_server:cast(Transport, close). + +%%-------------------------------------------------------------------- +%% Function: Read(Transport, Len) -> {ok, Data} +%% +%% Data = binary() +%% +%% Description: Reads data through from the wrapped transoprt +%%-------------------------------------------------------------------- +read(Transport, Len) when is_integer(Len) -> + gen_server:call(Transport, {read, Len}, _Timeout=10000). + +%%==================================================================== +%% gen_server callbacks +%%==================================================================== + +%%-------------------------------------------------------------------- +%% Function: init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% Description: Initiates the server +%%-------------------------------------------------------------------- +init([Wrapped]) -> + {ok, #buffered_transport{wrapped = Wrapped, + write_buffer = []}}. + +%%-------------------------------------------------------------------- +%% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | +%% {stop, Reason, State} +%% Description: Handling call messages +%%-------------------------------------------------------------------- +handle_call({write, Data}, _From, State = #buffered_transport{write_buffer = WBuf}) -> + {reply, ok, State#buffered_transport{write_buffer = [WBuf, Data]}}; + +handle_call({read, Len}, _From, State = #buffered_transport{wrapped = Wrapped}) -> + Response = thrift_transport:read(Wrapped, Len), + {reply, Response, State}; + +handle_call(flush, _From, State = #buffered_transport{write_buffer = WBuf, + wrapped = Wrapped}) -> + Response = thrift_transport:write(Wrapped, WBuf), + thrift_transport:flush(Wrapped), + {reply, Response, State#buffered_transport{write_buffer = []}}. + +%%-------------------------------------------------------------------- +%% Function: handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% Description: Handling cast messages +%%-------------------------------------------------------------------- +handle_cast(close, State = #buffered_transport{write_buffer = WBuf, + wrapped = Wrapped}) -> + thrift_transport:write(Wrapped, WBuf), + %% Wrapped is closed by terminate/2 + %% error_logger:info_msg("thrift_buffered_transport ~p: closing", [self()]), + {stop, normal, State}; +handle_cast(Msg, State=#buffered_transport{}) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% Function: handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% Description: Handling all non call/cast messages +%%-------------------------------------------------------------------- +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% Function: terminate(Reason, State) -> void() +%% Description: This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any necessary +%% cleaning up. When it returns, the gen_server terminates with Reason. +%% The return value is ignored. +%%-------------------------------------------------------------------- +terminate(_Reason, State = #buffered_transport{wrapped=Wrapped}) -> + thrift_transport:close(Wrapped), + ok. + +%%-------------------------------------------------------------------- +%% Func: code_change(OldVsn, State, Extra) -> {ok, NewState} +%% Description: Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +%%%% FACTORY GENERATION %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +new_transport_factory(WrapFactory) -> + F = fun() -> + {ok, Wrapped} = WrapFactory(), + new(Wrapped) + end, + {ok, F}. diff --git a/lib/erl/src/thrift_client.erl b/lib/erl/src/thrift_client.erl new file mode 100644 index 00000000..5ba8aee6 --- /dev/null +++ b/lib/erl/src/thrift_client.erl @@ -0,0 +1,330 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_client). + +-behaviour(gen_server). + +%% API +-export([start_link/2, start_link/3, start_link/4, call/3, send_call/3, close/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + + +-include("thrift_constants.hrl"). +-include("thrift_protocol.hrl"). + +-record(state, {service, protocol, seqid}). + +%%==================================================================== +%% API +%%==================================================================== +%%-------------------------------------------------------------------- +%% Function: start_link() -> {ok,Pid} | ignore | {error,Error} +%% Description: Starts the server +%%-------------------------------------------------------------------- +start_link(Host, Port, Service) when is_integer(Port), is_atom(Service) -> + start_link(Host, Port, Service, []). + + +%% +%% Splits client options into protocol options and transport options +%% +%% split_options([Options...]) -> {ProtocolOptions, TransportOptions} +%% +split_options(Options) -> + split_options(Options, [], []). + +split_options([], ProtoIn, TransIn) -> + {ProtoIn, TransIn}; + +split_options([Opt = {OptKey, _} | Rest], ProtoIn, TransIn) + when OptKey =:= strict_read; + OptKey =:= strict_write -> + split_options(Rest, [Opt | ProtoIn], TransIn); + +split_options([Opt = {OptKey, _} | Rest], ProtoIn, TransIn) + when OptKey =:= framed; + OptKey =:= connect_timeout; + OptKey =:= sockopts -> + split_options(Rest, ProtoIn, [Opt | TransIn]). + + +%% Backwards-compatible starter for the common-case of socket transports +start_link(Host, Port, Service, Options) + when is_integer(Port), is_atom(Service), is_list(Options) -> + {ProtoOpts, TransOpts} = split_options(Options), + + {ok, TransportFactory} = + thrift_socket_transport:new_transport_factory(Host, Port, TransOpts), + + {ok, ProtocolFactory} = thrift_binary_protocol:new_protocol_factory( + TransportFactory, ProtoOpts), + + start_link(ProtocolFactory, Service). + + +%% ProtocolFactory :: fun() -> thrift_protocol() +start_link(ProtocolFactory, Service) + when is_function(ProtocolFactory), is_atom(Service) -> + case gen_server:start_link(?MODULE, [Service], []) of + {ok, Pid} -> + case gen_server:call(Pid, {connect, ProtocolFactory}) of + ok -> + {ok, Pid}; + Error -> + Error + end; + Else -> + Else + end. + +call(Client, Function, Args) + when is_pid(Client), is_atom(Function), is_list(Args) -> + case gen_server:call(Client, {call, Function, Args}) of + R = {ok, _} -> R; + R = {error, _} -> R; + {exception, Exception} -> throw(Exception) + end. + +cast(Client, Function, Args) + when is_pid(Client), is_atom(Function), is_list(Args) -> + gen_server:cast(Client, {call, Function, Args}). + +%% Sends a function call but does not read the result. This is useful +%% if you're trying to log non-oneway function calls to write-only +%% transports like thrift_disk_log_transport. +send_call(Client, Function, Args) + when is_pid(Client), is_atom(Function), is_list(Args) -> + gen_server:call(Client, {send_call, Function, Args}). + +close(Client) when is_pid(Client) -> + gen_server:cast(Client, close). + +%%==================================================================== +%% gen_server callbacks +%%==================================================================== + +%%-------------------------------------------------------------------- +%% Function: init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% Description: Initiates the server +%%-------------------------------------------------------------------- +init([Service]) -> + {ok, #state{service = Service}}. + +%%-------------------------------------------------------------------- +%% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | +%% {stop, Reason, State} +%% Description: Handling call messages +%%-------------------------------------------------------------------- +handle_call({connect, ProtocolFactory}, _From, + State = #state{service = Service}) -> + case ProtocolFactory() of + {ok, Protocol} -> + {reply, ok, State#state{protocol = Protocol, + seqid = 0}}; + Error -> + {stop, normal, Error, State} + end; + +handle_call({call, Function, Args}, _From, State = #state{service = Service}) -> + Result = catch_function_exceptions( + fun() -> + ok = send_function_call(State, Function, Args), + receive_function_result(State, Function) + end, + Service), + {reply, Result, State}; + + +handle_call({send_call, Function, Args}, _From, State = #state{service = Service}) -> + Result = catch_function_exceptions( + fun() -> + send_function_call(State, Function, Args) + end, + Service), + {reply, Result, State}. + + +%% Helper function that catches exceptions thrown by sending or receiving +%% a function and returns the correct response for call or send_only above. +catch_function_exceptions(Fun, Service) -> + try + Fun() + catch + throw:{return, Return} -> + Return; + error:function_clause -> + ST = erlang:get_stacktrace(), + case hd(ST) of + {Service, function_info, [Function, _]} -> + {error, {no_function, Function}}; + _ -> throw({error, {function_clause, ST}}) + end + end. + + +%%-------------------------------------------------------------------- +%% Function: handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% Description: Handling cast messages +%%-------------------------------------------------------------------- +handle_cast({call, Function, Args}, State = #state{service = Service, + protocol = Protocol, + seqid = SeqId}) -> + _Result = + try + ok = send_function_call(State, Function, Args), + receive_function_result(State, Function) + catch + Class:Reason -> + error_logger:error_msg("error ignored in handle_cast({cast,...},...): ~p:~p~n", [Class, Reason]) + end, + + {noreply, State}; + +handle_cast(close, State=#state{protocol = Protocol}) -> +%% error_logger:info_msg("thrift_client ~p received close", [self()]), + {stop,normal,State}; +handle_cast(_Msg, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% Function: handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% Description: Handling all non call/cast messages +%%-------------------------------------------------------------------- +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% Function: terminate(Reason, State) -> void() +%% Description: This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any necessary +%% cleaning up. When it returns, the gen_server terminates with Reason. +%% The return value is ignored. +%%-------------------------------------------------------------------- +terminate(Reason, State = #state{protocol=undefined}) -> + ok; +terminate(Reason, State = #state{protocol=Protocol}) -> + thrift_protocol:close_transport(Protocol), + ok. + +%%-------------------------------------------------------------------- +%% Func: code_change(OldVsn, State, Extra) -> {ok, NewState} +%% Description: Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +send_function_call(#state{protocol = Proto, + service = Service, + seqid = SeqId}, + Function, + Args) -> + Params = Service:function_info(Function, params_type), + {struct, PList} = Params, + if + length(PList) =/= length(Args) -> + throw({return, {error, {bad_args, Function, Args}}}); + true -> ok + end, + + Begin = #protocol_message_begin{name = atom_to_list(Function), + type = ?tMessageType_CALL, + seqid = SeqId}, + ok = thrift_protocol:write(Proto, Begin), + ok = thrift_protocol:write(Proto, {Params, list_to_tuple([Function | Args])}), + ok = thrift_protocol:write(Proto, message_end), + thrift_protocol:flush_transport(Proto), + ok. + +receive_function_result(State = #state{protocol = Proto, + service = Service}, + Function) -> + ResultType = Service:function_info(Function, reply_type), + read_result(State, Function, ResultType). + +read_result(_State, + _Function, + oneway_void) -> + {ok, ok}; + +read_result(State = #state{protocol = Proto, + seqid = SeqId}, + Function, + ReplyType) -> + case thrift_protocol:read(Proto, message_begin) of + #protocol_message_begin{seqid = RetSeqId} when RetSeqId =/= SeqId -> + {error, {bad_seq_id, SeqId}}; + + #protocol_message_begin{type = ?tMessageType_EXCEPTION} -> + handle_application_exception(State); + + #protocol_message_begin{type = ?tMessageType_REPLY} -> + handle_reply(State, Function, ReplyType) + end. + +handle_reply(State = #state{protocol = Proto, + service = Service}, + Function, + ReplyType) -> + {struct, ExceptionFields} = Service:function_info(Function, exceptions), + ReplyStructDef = {struct, [{0, ReplyType}] ++ ExceptionFields}, + {ok, Reply} = thrift_protocol:read(Proto, ReplyStructDef), + ReplyList = tuple_to_list(Reply), + true = length(ReplyList) == length(ExceptionFields) + 1, + ExceptionVals = tl(ReplyList), + Thrown = [X || X <- ExceptionVals, + X =/= undefined], + Result = + case Thrown of + [] when ReplyType == {struct, []} -> + {ok, ok}; + [] -> + {ok, hd(ReplyList)}; + [Exception] -> + {exception, Exception} + end, + ok = thrift_protocol:read(Proto, message_end), + Result. + +handle_application_exception(State = #state{protocol = Proto}) -> + {ok, Exception} = thrift_protocol:read(Proto, + ?TApplicationException_Structure), + ok = thrift_protocol:read(Proto, message_end), + XRecord = list_to_tuple( + ['TApplicationException' | tuple_to_list(Exception)]), + error_logger:error_msg("X: ~p~n", [XRecord]), + true = is_record(XRecord, 'TApplicationException'), + {exception, XRecord}. diff --git a/lib/erl/src/thrift_disk_log_transport.erl b/lib/erl/src/thrift_disk_log_transport.erl new file mode 100644 index 00000000..761fa309 --- /dev/null +++ b/lib/erl/src/thrift_disk_log_transport.erl @@ -0,0 +1,118 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +%%% Todo: this might be better off as a gen_server type of transport +%%% that handles stuff like group commit, similar to TFileTransport +%%% in cpp land +-module(thrift_disk_log_transport). + +-behaviour(thrift_transport). + +%% API +-export([new/2, new_transport_factory/2, new_transport_factory/3]). + +%% thrift_transport callbacks +-export([read/2, write/2, force_flush/1, flush/1, close/1]). + +%% state +-record(dl_transport, {log, + close_on_close = false, + sync_every = infinity, + sync_tref}). + + +%% Create a transport attached to an already open log. +%% If you'd like this transport to close the disk_log using disk_log:lclose() +%% when the transport is closed, pass a {close_on_close, true} tuple in the +%% Opts list. +new(LogName, Opts) when is_atom(LogName), is_list(Opts) -> + State = parse_opts(Opts, #dl_transport{log = LogName}), + + State2 = + case State#dl_transport.sync_every of + N when is_integer(N), N > 0 -> + {ok, TRef} = timer:apply_interval(N, ?MODULE, force_flush, State), + State#dl_transport{sync_tref = TRef}; + _ -> State + end, + + thrift_transport:new(?MODULE, State2). + + +parse_opts([], State) -> + State; +parse_opts([{close_on_close, Bool} | Rest], State) when is_boolean(Bool) -> + State#dl_transport{close_on_close = Bool}; +parse_opts([{sync_every, Int} | Rest], State) when is_integer(Int), Int > 0 -> + State#dl_transport{sync_every = Int}. + + +%%%% TRANSPORT IMPLENTATION %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%% disk_log_transport is write-only +read(_State, Len) -> + {error, no_read_from_disk_log}. + +write(#dl_transport{log = Log}, Data) -> + disk_log:balog(Log, erlang:iolist_to_binary(Data)). + +force_flush(#dl_transport{log = Log}) -> + error_logger:info_msg("~p syncing~n", [?MODULE]), + disk_log:sync(Log). + +flush(#dl_transport{log = Log, sync_every = SE}) -> + case SE of + undefined -> % no time-based sync + disk_log:sync(Log); + _Else -> % sync will happen automagically + ok + end. + + +%% On close, close the underlying log if we're configured to do so. +close(#dl_transport{close_on_close = false}) -> + ok; +close(#dl_transport{log = Log}) -> + disk_log:lclose(Log). + + +%%%% FACTORY GENERATION %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +new_transport_factory(Name, ExtraLogOpts) -> + new_transport_factory(Name, ExtraLogOpts, [{close_on_close, true}, + {sync_every, 500}]). + +new_transport_factory(Name, ExtraLogOpts, TransportOpts) -> + F = fun() -> factory_impl(Name, ExtraLogOpts, TransportOpts) end, + {ok, F}. + +factory_impl(Name, ExtraLogOpts, TransportOpts) -> + LogOpts = [{name, Name}, + {format, external}, + {type, wrap} | + ExtraLogOpts], + Log = + case disk_log:open(LogOpts) of + {ok, Log} -> + Log; + {repaired, Log, Info1, Info2} -> + error_logger:info_msg("Disk log ~p repaired: ~p, ~p~n", [Log, Info1, Info2]), + Log + end, + new(Log, TransportOpts). diff --git a/lib/erl/src/thrift_file_transport.erl b/lib/erl/src/thrift_file_transport.erl new file mode 100644 index 00000000..5ac2dbe1 --- /dev/null +++ b/lib/erl/src/thrift_file_transport.erl @@ -0,0 +1,87 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_file_transport). + +-behaviour(thrift_transport). + +-export([new_reader/1, + new/1, + new/2, + write/2, read/2, flush/1, close/1]). + +-record(t_file_transport, {device, + should_close = true, + mode = write}). + +%%%% CONSTRUCTION %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +new_reader(Filename) -> + case file:open(Filename, [read, binary, {read_ahead, 1024*1024}]) of + {ok, IODevice} -> + new(IODevice, [{should_close, true}, {mode, read}]); + Error -> Error + end. + +new(Device) -> + new(Device, []). + +%% Device :: io_device() +%% +%% Device should be opened in raw and binary mode. +new(Device, Opts) when is_list(Opts) -> + State = parse_opts(Opts, #t_file_transport{device = Device}), + thrift_transport:new(?MODULE, State). + + +%% Parse options +parse_opts([{should_close, Bool} | Rest], State) when is_boolean(Bool) -> + parse_opts(Rest, State#t_file_transport{should_close = Bool}); +parse_opts([{mode, Mode} | Rest], State) + when Mode =:= write; + Mode =:= read -> + parse_opts(Rest, State#t_file_transport{mode = Mode}); +parse_opts([], State) -> + State. + + +%%%% TRANSPORT IMPL %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +write(#t_file_transport{device = Device, mode = write}, Data) -> + file:write(Device, Data); +write(_T, _D) -> + {error, read_mode}. + + +read(#t_file_transport{device = Device, mode = read}, Len) + when is_integer(Len), Len >= 0 -> + file:read(Device, Len); +read(_T, _D) -> + {error, read_mode}. + +flush(#t_file_transport{device = Device, mode = write}) -> + file:sync(Device). + +close(#t_file_transport{device = Device, should_close = SC}) -> + case SC of + true -> + file:close(Device); + false -> + ok + end. diff --git a/lib/erl/src/thrift_framed_transport.erl b/lib/erl/src/thrift_framed_transport.erl new file mode 100644 index 00000000..01bab70b --- /dev/null +++ b/lib/erl/src/thrift_framed_transport.erl @@ -0,0 +1,208 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_framed_transport). + +-behaviour(gen_server). +-behaviour(thrift_transport). + +%% API +-export([new/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +%% thrift_transport callbacks +-export([write/2, read/2, flush/1, close/1]). + +-record(framed_transport, {wrapped, % a thrift_transport + read_buffer, % iolist() + write_buffer % iolist() + }). + +%%==================================================================== +%% API +%%==================================================================== +%%-------------------------------------------------------------------- +%% Function: start_link() -> {ok,Pid} | ignore | {error,Error} +%% Description: Starts the server +%%-------------------------------------------------------------------- +new(WrappedTransport) -> + case gen_server:start_link(?MODULE, [WrappedTransport], []) of + {ok, Pid} -> + thrift_transport:new(?MODULE, Pid); + Else -> + Else + end. + +%%-------------------------------------------------------------------- +%% Function: write(Transport, Data) -> ok +%% +%% Data = iolist() +%% +%% Description: Writes data into the buffer +%%-------------------------------------------------------------------- +write(Transport, Data) -> + gen_server:call(Transport, {write, Data}). + +%%-------------------------------------------------------------------- +%% Function: flush(Transport) -> ok +%% +%% Description: Flushes the buffer through to the wrapped transport +%%-------------------------------------------------------------------- +flush(Transport) -> + gen_server:call(Transport, flush). + +%%-------------------------------------------------------------------- +%% Function: close(Transport) -> ok +%% +%% Description: Closes the transport and the wrapped transport +%%-------------------------------------------------------------------- +close(Transport) -> + gen_server:cast(Transport, close). + +%%-------------------------------------------------------------------- +%% Function: Read(Transport, Len) -> {ok, Data} +%% +%% Data = binary() +%% +%% Description: Reads data through from the wrapped transoprt +%%-------------------------------------------------------------------- +read(Transport, Len) when is_integer(Len) -> + gen_server:call(Transport, {read, Len}). + +%%==================================================================== +%% gen_server callbacks +%%==================================================================== + +%%-------------------------------------------------------------------- +%% Function: init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% Description: Initiates the server +%%-------------------------------------------------------------------- +init([Wrapped]) -> + {ok, #framed_transport{wrapped = Wrapped, + read_buffer = [], + write_buffer = []}}. + +%%-------------------------------------------------------------------- +%% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | +%% {stop, Reason, State} +%% Description: Handling call messages +%%-------------------------------------------------------------------- +handle_call({write, Data}, _From, State = #framed_transport{write_buffer = WBuf}) -> + {reply, ok, State#framed_transport{write_buffer = [WBuf, Data]}}; + +handle_call({read, Len}, _From, State = #framed_transport{wrapped = Wrapped, + read_buffer = RBuf}) -> + {RBuf1, RBuf1Size} = + %% if the read buffer is empty, read another frame + %% otherwise, just read from what's left in the buffer + case iolist_size(RBuf) of + 0 -> + %% read the frame length + {ok, <>} = + thrift_transport:read(Wrapped, 4), + %% then read the data + {ok, Bin} = + thrift_transport:read(Wrapped, FrameLen), + {Bin, erlang:byte_size(Bin)}; + Sz -> + {RBuf, Sz} + end, + + %% pull off Give bytes, return them to the user, leave the rest in the buffer + Give = min(RBuf1Size, Len), + <> = iolist_to_binary(RBuf1), + + Response = {ok, Data}, + State1 = State#framed_transport{read_buffer=RBuf2}, + + {reply, Response, State1}; + +handle_call(flush, _From, State) -> + {Response, State1} = do_flush(State), + {reply, Response, State1}. + +%%-------------------------------------------------------------------- +%% Function: handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% Description: Handling cast messages +%%-------------------------------------------------------------------- +handle_cast(close, State) -> + {_, State1} = do_flush(State), + %% Wrapped is closed by terminate/2 + %% error_logger:info_msg("thrift_framed_transport ~p: closing", [self()]), + {stop, normal, State}; +handle_cast(Msg, State=#framed_transport{}) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% Function: handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% Description: Handling all non call/cast messages +%%-------------------------------------------------------------------- +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% Function: terminate(Reason, State) -> void() +%% Description: This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any necessary +%% cleaning up. When it returns, the gen_server terminates with Reason. +%% The return value is ignored. +%%-------------------------------------------------------------------- +terminate(_Reason, State = #framed_transport{wrapped=Wrapped}) -> + thrift_transport:close(Wrapped), + ok. + +%%-------------------------------------------------------------------- +%% Func: code_change(OldVsn, State, Extra) -> {ok, NewState} +%% Description: Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +do_flush(State = #framed_transport{write_buffer = Buffer, + wrapped = Wrapped}) -> + FrameLen = iolist_size(Buffer), + Data = [<>, Buffer], + + Response = thrift_transport:write(Wrapped, Data), + + thrift_transport:flush(Wrapped), + + State1 = State#framed_transport{write_buffer = []}, + {Response, State1}. + +min(A,B) when A A; +min(_,B) -> B. + diff --git a/lib/erl/src/thrift_http_transport.erl b/lib/erl/src/thrift_http_transport.erl new file mode 100644 index 00000000..f8c18277 --- /dev/null +++ b/lib/erl/src/thrift_http_transport.erl @@ -0,0 +1,199 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_http_transport). + +-behaviour(gen_server). +-behaviour(thrift_transport). + +%% API +-export([new/2, new/3]). + +%% gen_server callbacks +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +%% thrift_transport callbacks +-export([write/2, read/2, flush/1, close/1]). + +-record(http_transport, {host, % string() + path, % string() + read_buffer, % iolist() + write_buffer, % iolist() + http_options, % see http(3) + extra_headers % [{str(), str()}, ...] + }). + +%%==================================================================== +%% API +%%==================================================================== +%%-------------------------------------------------------------------- +%% Function: new() -> {ok, Transport} | ignore | {error,Error} +%% Description: Starts the server +%%-------------------------------------------------------------------- +new(Host, Path) -> + new(Host, Path, _Options = []). + +%%-------------------------------------------------------------------- +%% Options include: +%% {http_options, HttpOptions} = See http(3) +%% {extra_headers, ExtraHeaders} = List of extra HTTP headers +%%-------------------------------------------------------------------- +new(Host, Path, Options) -> + case gen_server:start_link(?MODULE, {Host, Path, Options}, []) of + {ok, Pid} -> + thrift_transport:new(?MODULE, Pid); + Else -> + Else + end. + +%%-------------------------------------------------------------------- +%% Function: write(Transport, Data) -> ok +%% +%% Data = iolist() +%% +%% Description: Writes data into the buffer +%%-------------------------------------------------------------------- +write(Transport, Data) -> + gen_server:call(Transport, {write, Data}). + +%%-------------------------------------------------------------------- +%% Function: flush(Transport) -> ok +%% +%% Description: Flushes the buffer, making a request +%%-------------------------------------------------------------------- +flush(Transport) -> + gen_server:call(Transport, flush). + +%%-------------------------------------------------------------------- +%% Function: close(Transport) -> ok +%% +%% Description: Closes the transport +%%-------------------------------------------------------------------- +close(Transport) -> + gen_server:cast(Transport, close). + +%%-------------------------------------------------------------------- +%% Function: Read(Transport, Len) -> {ok, Data} +%% +%% Data = binary() +%% +%% Description: Reads data through from the wrapped transoprt +%%-------------------------------------------------------------------- +read(Transport, Len) when is_integer(Len) -> + gen_server:call(Transport, {read, Len}). + +%%==================================================================== +%% gen_server callbacks +%%==================================================================== + +init({Host, Path, Options}) -> + State1 = #http_transport{host = Host, + path = Path, + read_buffer = [], + write_buffer = [], + http_options = [], + extra_headers = []}, + ApplyOption = + fun + ({http_options, HttpOpts}, State = #http_transport{}) -> + State#http_transport{http_options = HttpOpts}; + ({extra_headers, ExtraHeaders}, State = #http_transport{}) -> + State#http_transport{extra_headers = ExtraHeaders}; + (Other, #http_transport{}) -> + {invalid_option, Other}; + (_, Error) -> + Error + end, + case lists:foldl(ApplyOption, State1, Options) of + State2 = #http_transport{} -> + {ok, State2}; + Else -> + {stop, Else} + end. + +handle_call({write, Data}, _From, State = #http_transport{write_buffer = WBuf}) -> + {reply, ok, State#http_transport{write_buffer = [WBuf, Data]}}; + +handle_call({read, Len}, _From, State = #http_transport{read_buffer = RBuf}) -> + %% Pull off Give bytes, return them to the user, leave the rest in the buffer. + Give = min(iolist_size(RBuf), Len), + case iolist_to_binary(RBuf) of + <> -> + Response = {ok, Data}, + State1 = State#http_transport{read_buffer=RBuf1}, + {reply, Response, State1}; + _ -> + {reply, {error, 'EOF'}, State} + end; + +handle_call(flush, _From, State) -> + {Response, State1} = do_flush(State), + {reply, Response, State1}. + +handle_cast(close, State) -> + {_, State1} = do_flush(State), + {stop, normal, State1}; + +handle_cast(_Msg, State=#http_transport{}) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +do_flush(State = #http_transport{host = Host, + path = Path, + read_buffer = Rbuf, + write_buffer = Wbuf, + http_options = HttpOptions, + extra_headers = ExtraHeaders}) -> + case iolist_to_binary(Wbuf) of + <<>> -> + %% Don't bother flushing empty buffers. + {ok, State}; + WBinary -> + {ok, {{_Version, 200, _ReasonPhrase}, _Headers, Body}} = + http:request(post, + {"http://" ++ Host ++ Path, + [{"User-Agent", "Erlang/thrift_http_transport"} | ExtraHeaders], + "application/x-thrift", + WBinary}, + HttpOptions, + [{body_format, binary}]), + + State1 = State#http_transport{read_buffer = [Rbuf, Body], + write_buffer = []}, + {ok, State1} + end. + +min(A,B) when A A; +min(_,B) -> B. diff --git a/lib/erl/src/thrift_memory_buffer.erl b/lib/erl/src/thrift_memory_buffer.erl new file mode 100644 index 00000000..b4f607a9 --- /dev/null +++ b/lib/erl/src/thrift_memory_buffer.erl @@ -0,0 +1,164 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_memory_buffer). + +-behaviour(gen_server). +-behaviour(thrift_transport). + +%% API +-export([new/0, new_transport_factory/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +%% thrift_transport callbacks +-export([write/2, read/2, flush/1, close/1]). + +-record(memory_buffer, {buffer}). + +%%==================================================================== +%% API +%%==================================================================== +new() -> + case gen_server:start_link(?MODULE, [], []) of + {ok, Pid} -> + thrift_transport:new(?MODULE, Pid); + Else -> + Else + end. + +new_transport_factory() -> + {ok, fun() -> new() end}. + +%%-------------------------------------------------------------------- +%% Function: write(Transport, Data) -> ok +%% +%% Data = iolist() +%% +%% Description: Writes data into the buffer +%%-------------------------------------------------------------------- +write(Transport, Data) -> + gen_server:call(Transport, {write, Data}). + +%%-------------------------------------------------------------------- +%% Function: flush(Transport) -> ok +%% +%% Description: Flushes the buffer through to the wrapped transport +%%-------------------------------------------------------------------- +flush(Transport) -> + gen_server:call(Transport, flush). + +%%-------------------------------------------------------------------- +%% Function: close(Transport) -> ok +%% +%% Description: Closes the transport and the wrapped transport +%%-------------------------------------------------------------------- +close(Transport) -> + gen_server:cast(Transport, close). + +%%-------------------------------------------------------------------- +%% Function: Read(Transport, Len) -> {ok, Data} +%% +%% Data = binary() +%% +%% Description: Reads data through from the wrapped transoprt +%%-------------------------------------------------------------------- +read(Transport, Len) when is_integer(Len) -> + gen_server:call(Transport, {read, Len}). + +%%==================================================================== +%% gen_server callbacks +%%==================================================================== + +%%-------------------------------------------------------------------- +%% Function: init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% Description: Initiates the server +%%-------------------------------------------------------------------- +init([]) -> + {ok, #memory_buffer{buffer = []}}. + +%%-------------------------------------------------------------------- +%% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | +%% {stop, Reason, State} +%% Description: Handling call messages +%%-------------------------------------------------------------------- +handle_call({write, Data}, _From, State = #memory_buffer{buffer = Buf}) -> + {reply, ok, State#memory_buffer{buffer = [Buf, Data]}}; + +handle_call({read, Len}, _From, State = #memory_buffer{buffer = Buf}) -> + Binary = iolist_to_binary(Buf), + Give = min(iolist_size(Binary), Len), + {Result, Remaining} = split_binary(Binary, Give), + {reply, {ok, Result}, State#memory_buffer{buffer = Remaining}}; + +handle_call(flush, _From, State) -> + {reply, ok, State}. + +%%-------------------------------------------------------------------- +%% Function: handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% Description: Handling cast messages +%%-------------------------------------------------------------------- +handle_cast(close, State) -> + {stop, normal, State}; +handle_cast(Msg, State=#memory_buffer{}) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% Function: handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% Description: Handling all non call/cast messages +%%-------------------------------------------------------------------- +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% Function: terminate(Reason, State) -> void() +%% Description: This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any necessary +%% cleaning up. When it returns, the gen_server terminates with Reason. +%% The return value is ignored. +%%-------------------------------------------------------------------- +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% Func: code_change(OldVsn, State, Extra) -> {ok, NewState} +%% Description: Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +min(A,B) when A A; +min(_,B) -> B. + diff --git a/lib/erl/src/thrift_processor.erl b/lib/erl/src/thrift_processor.erl new file mode 100644 index 00000000..e26fb330 --- /dev/null +++ b/lib/erl/src/thrift_processor.erl @@ -0,0 +1,188 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_processor). + +-export([init/1]). + +-include("thrift_constants.hrl"). +-include("thrift_protocol.hrl"). + +-record(thrift_processor, {handler, in_protocol, out_protocol, service}). + +init({Server, ProtoGen, Service, Handler}) when is_function(ProtoGen, 0) -> + {ok, IProt, OProt} = ProtoGen(), + loop(#thrift_processor{in_protocol = IProt, + out_protocol = OProt, + service = Service, + handler = Handler}). + +loop(State = #thrift_processor{in_protocol = IProto, + out_protocol = OProto}) -> + case thrift_protocol:read(IProto, message_begin) of + #protocol_message_begin{name = Function, + type = ?tMessageType_CALL} -> + ok = handle_function(State, list_to_atom(Function)), + loop(State); + #protocol_message_begin{name = Function, + type = ?tMessageType_ONEWAY} -> + ok = handle_function(State, list_to_atom(Function)), + loop(State); + {error, timeout} -> + thrift_protocol:close_transport(OProto), + ok; + {error, closed} -> + %% error_logger:info_msg("Client disconnected~n"), + thrift_protocol:close_transport(OProto), + exit(shutdown) + end. + +handle_function(State=#thrift_processor{in_protocol = IProto, + out_protocol = OProto, + handler = Handler, + service = Service}, + Function) -> + InParams = Service:function_info(Function, params_type), + + {ok, Params} = thrift_protocol:read(IProto, InParams), + + try + Result = Handler:handle_function(Function, Params), + %% {Micro, Result} = better_timer(Handler, handle_function, [Function, Params]), + %% error_logger:info_msg("Processed ~p(~p) in ~.4fms~n", + %% [Function, Params, Micro/1000.0]), + handle_success(State, Function, Result) + catch + Type:Data -> + handle_function_catch(State, Function, Type, Data) + end, + after_reply(OProto). + +handle_function_catch(State = #thrift_processor{service = Service}, + Function, ErrType, ErrData) -> + IsOneway = Service:function_info(Function, reply_type) =:= oneway_void, + + case {ErrType, ErrData} of + _ when IsOneway -> + Stack = erlang:get_stacktrace(), + error_logger:warning_msg( + "oneway void ~p threw error which must be ignored: ~p", + [Function, {ErrType, ErrData, Stack}]), + ok; + + {throw, Exception} when is_tuple(Exception), size(Exception) > 0 -> + error_logger:warning_msg("~p threw exception: ~p~n", [Function, Exception]), + handle_exception(State, Function, Exception), + ok; % we still want to accept more requests from this client + + {error, Error} -> + ok = handle_error(State, Function, Error) + end. + +handle_success(State = #thrift_processor{out_protocol = OProto, + service = Service}, + Function, + Result) -> + ReplyType = Service:function_info(Function, reply_type), + StructName = atom_to_list(Function) ++ "_result", + + ok = case Result of + {reply, ReplyData} -> + Reply = {{struct, [{0, ReplyType}]}, {StructName, ReplyData}}, + send_reply(OProto, Function, ?tMessageType_REPLY, Reply); + + ok when ReplyType == {struct, []} -> + send_reply(OProto, Function, ?tMessageType_REPLY, {ReplyType, {StructName}}); + + ok when ReplyType == oneway_void -> + %% no reply for oneway void + ok + end. + +handle_exception(State = #thrift_processor{out_protocol = OProto, + service = Service}, + Function, + Exception) -> + ExceptionType = element(1, Exception), + %% Fetch a structure like {struct, [{-2, {struct, {Module, Type}}}, + %% {-3, {struct, {Module, Type}}}]} + + ReplySpec = Service:function_info(Function, exceptions), + {struct, XInfo} = ReplySpec, + + true = is_list(XInfo), + + %% Assuming we had a type1 exception, we'd get: [undefined, Exception, undefined] + %% e.g.: [{-1, type0}, {-2, type1}, {-3, type2}] + ExceptionList = [case Type of + ExceptionType -> Exception; + _ -> undefined + end + || {_Fid, {struct, {_Module, Type}}} <- XInfo], + + ExceptionTuple = list_to_tuple([Function | ExceptionList]), + + % Make sure we got at least one defined + case lists:all(fun(X) -> X =:= undefined end, ExceptionList) of + true -> + ok = handle_unknown_exception(State, Function, Exception); + false -> + ok = send_reply(OProto, Function, ?tMessageType_REPLY, {ReplySpec, ExceptionTuple}) + end. + +%% +%% Called when an exception has been explicitly thrown by the service, but it was +%% not one of the exceptions that was defined for the function. +%% +handle_unknown_exception(State, Function, Exception) -> + handle_error(State, Function, {exception_not_declared_as_thrown, + Exception}). + +handle_error(#thrift_processor{out_protocol = OProto}, Function, Error) -> + Stack = erlang:get_stacktrace(), + error_logger:error_msg("~p had an error: ~p~n", [Function, {Error, Stack}]), + + Message = + case application:get_env(thrift, exceptions_include_traces) of + {ok, true} -> + lists:flatten(io_lib:format("An error occurred: ~p~n", + [{Error, Stack}])); + _ -> + "An unknown handler error occurred." + end, + Reply = {?TApplicationException_Structure, + #'TApplicationException'{ + message = Message, + type = ?TApplicationException_UNKNOWN}}, + send_reply(OProto, Function, ?tMessageType_EXCEPTION, Reply). + +send_reply(OProto, Function, ReplyMessageType, Reply) -> + ok = thrift_protocol:write(OProto, #protocol_message_begin{ + name = atom_to_list(Function), + type = ReplyMessageType, + seqid = 0}), + ok = thrift_protocol:write(OProto, Reply), + ok = thrift_protocol:write(OProto, message_end), + ok = thrift_protocol:flush_transport(OProto), + ok. + +after_reply(OProto) -> + ok = thrift_protocol:flush_transport(OProto) + %% ok = thrift_protocol:close_transport(OProto) + . diff --git a/lib/erl/src/thrift_protocol.erl b/lib/erl/src/thrift_protocol.erl new file mode 100644 index 00000000..1bfb0a42 --- /dev/null +++ b/lib/erl/src/thrift_protocol.erl @@ -0,0 +1,356 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_protocol). + +-export([new/2, + write/2, + read/2, + read/3, + skip/2, + flush_transport/1, + close_transport/1, + typeid_to_atom/1 + ]). + +-export([behaviour_info/1]). + +-include("thrift_constants.hrl"). +-include("thrift_protocol.hrl"). + +-record(protocol, {module, data}). + +behaviour_info(callbacks) -> + [ + {read, 2}, + {write, 2}, + {flush_transport, 1}, + {close_transport, 1} + ]; +behaviour_info(_Else) -> undefined. + +new(Module, Data) when is_atom(Module) -> + {ok, #protocol{module = Module, + data = Data}}. + +flush_transport(#protocol{module = Module, + data = Data}) -> + Module:flush_transport(Data). + +close_transport(#protocol{module = Module, + data = Data}) -> + Module:close_transport(Data). + +typeid_to_atom(?tType_STOP) -> field_stop; +typeid_to_atom(?tType_VOID) -> void; +typeid_to_atom(?tType_BOOL) -> bool; +typeid_to_atom(?tType_BYTE) -> byte; +typeid_to_atom(?tType_DOUBLE) -> double; +typeid_to_atom(?tType_I16) -> i16; +typeid_to_atom(?tType_I32) -> i32; +typeid_to_atom(?tType_I64) -> i64; +typeid_to_atom(?tType_STRING) -> string; +typeid_to_atom(?tType_STRUCT) -> struct; +typeid_to_atom(?tType_MAP) -> map; +typeid_to_atom(?tType_SET) -> set; +typeid_to_atom(?tType_LIST) -> list. + +term_to_typeid(void) -> ?tType_VOID; +term_to_typeid(bool) -> ?tType_BOOL; +term_to_typeid(byte) -> ?tType_BYTE; +term_to_typeid(double) -> ?tType_DOUBLE; +term_to_typeid(i16) -> ?tType_I16; +term_to_typeid(i32) -> ?tType_I32; +term_to_typeid(i64) -> ?tType_I64; +term_to_typeid(string) -> ?tType_STRING; +term_to_typeid({struct, _}) -> ?tType_STRUCT; +term_to_typeid({map, _, _}) -> ?tType_MAP; +term_to_typeid({set, _}) -> ?tType_SET; +term_to_typeid({list, _}) -> ?tType_LIST. + +%% Structure is like: +%% [{Fid, Type}, ...] +read(IProto, {struct, Structure}, Tag) + when is_list(Structure), is_atom(Tag) -> + + % If we want a tagged tuple, we need to offset all the tuple indices + % by 1 to avoid overwriting the tag. + Offset = if Tag =/= undefined -> 1; true -> 0 end, + IndexList = case length(Structure) of + N when N > 0 -> lists:seq(1 + Offset, N + Offset); + _ -> [] + end, + + SWithIndices = [{Fid, {Type, Index}} || + {{Fid, Type}, Index} <- + lists:zip(Structure, IndexList)], + % Fid -> {Type, Index} + SDict = dict:from_list(SWithIndices), + + ok = read(IProto, struct_begin), + RTuple0 = erlang:make_tuple(length(Structure) + Offset, undefined), + RTuple1 = if Tag =/= undefined -> setelement(1, RTuple0, Tag); + true -> RTuple0 + end, + + RTuple2 = read_struct_loop(IProto, SDict, RTuple1), + {ok, RTuple2}. + +read(IProto, {struct, {Module, StructureName}}) when is_atom(Module), + is_atom(StructureName) -> + read(IProto, Module:struct_info(StructureName), StructureName); + +read(IProto, S={struct, Structure}) when is_list(Structure) -> + read(IProto, S, undefined); + +read(IProto, {list, Type}) -> + #protocol_list_begin{etype = EType, size = Size} = + read(IProto, list_begin), + List = [Result || {ok, Result} <- + [read(IProto, Type) || _X <- lists:duplicate(Size, 0)]], + ok = read(IProto, list_end), + {ok, List}; + +read(IProto, {map, KeyType, ValType}) -> + #protocol_map_begin{size = Size} = + read(IProto, map_begin), + + List = [{Key, Val} || {{ok, Key}, {ok, Val}} <- + [{read(IProto, KeyType), + read(IProto, ValType)} || _X <- lists:duplicate(Size, 0)]], + ok = read(IProto, map_end), + {ok, dict:from_list(List)}; + +read(IProto, {set, Type}) -> + #protocol_set_begin{etype = _EType, + size = Size} = + read(IProto, set_begin), + List = [Result || {ok, Result} <- + [read(IProto, Type) || _X <- lists:duplicate(Size, 0)]], + ok = read(IProto, set_end), + {ok, sets:from_list(List)}; + +read(#protocol{module = Module, + data = ModuleData}, ProtocolType) -> + Module:read(ModuleData, ProtocolType). + +read_struct_loop(IProto, SDict, RTuple) -> + #protocol_field_begin{type = FType, id = Fid, name = Name} = + thrift_protocol:read(IProto, field_begin), + case {FType, Fid} of + {?tType_STOP, _} -> + RTuple; + _Else -> + case dict:find(Fid, SDict) of + {ok, {Type, Index}} -> + case term_to_typeid(Type) of + FType -> + {ok, Val} = read(IProto, Type), + thrift_protocol:read(IProto, field_end), + NewRTuple = setelement(Index, RTuple, Val), + read_struct_loop(IProto, SDict, NewRTuple); + Expected -> + error_logger:info_msg( + "Skipping field ~p with wrong type (~p != ~p)~n", + [Fid, FType, Expected]), + skip_field(FType, IProto, SDict, RTuple) + end; + _Else2 -> + error_logger:info_msg("Skipping field ~p with unknown fid~n", [Fid]), + skip_field(FType, IProto, SDict, RTuple) + end + end. + +skip_field(FType, IProto, SDict, RTuple) -> + FTypeAtom = thrift_protocol:typeid_to_atom(FType), + thrift_protocol:skip(IProto, FTypeAtom), + read(IProto, field_end), + read_struct_loop(IProto, SDict, RTuple). + + +skip(Proto, struct) -> + ok = read(Proto, struct_begin), + ok = skip_struct_loop(Proto), + ok = read(Proto, struct_end); + +skip(Proto, map) -> + Map = read(Proto, map_begin), + ok = skip_map_loop(Proto, Map), + ok = read(Proto, map_end); + +skip(Proto, set) -> + Set = read(Proto, set_begin), + ok = skip_set_loop(Proto, Set), + ok = read(Proto, set_end); + +skip(Proto, list) -> + List = read(Proto, list_begin), + ok = skip_list_loop(Proto, List), + ok = read(Proto, list_end); + +skip(Proto, Type) when is_atom(Type) -> + _Ignore = read(Proto, Type), + ok. + + +skip_struct_loop(Proto) -> + #protocol_field_begin{type = Type} = read(Proto, field_begin), + case Type of + ?tType_STOP -> + ok; + _Else -> + skip(Proto, Type), + ok = read(Proto, field_end), + skip_struct_loop(Proto) + end. + +skip_map_loop(Proto, Map = #protocol_map_begin{ktype = Ktype, + vtype = Vtype, + size = Size}) -> + case Size of + N when N > 0 -> + skip(Proto, Ktype), + skip(Proto, Vtype), + skip_map_loop(Proto, + Map#protocol_map_begin{size = Size - 1}); + 0 -> ok + end. + +skip_set_loop(Proto, Map = #protocol_set_begin{etype = Etype, + size = Size}) -> + case Size of + N when N > 0 -> + skip(Proto, Etype), + skip_set_loop(Proto, + Map#protocol_set_begin{size = Size - 1}); + 0 -> ok + end. + +skip_list_loop(Proto, Map = #protocol_list_begin{etype = Etype, + size = Size}) -> + case Size of + N when N > 0 -> + skip(Proto, Etype), + skip_list_loop(Proto, + Map#protocol_list_begin{size = Size - 1}); + 0 -> ok + end. + + +%%-------------------------------------------------------------------- +%% Function: write(OProto, {Type, Data}) -> ok +%% +%% Type = {struct, StructDef} | +%% {list, Type} | +%% {map, KeyType, ValType} | +%% {set, Type} | +%% BaseType +%% +%% Data = +%% tuple() -- for struct +%% | list() -- for list +%% | dictionary() -- for map +%% | set() -- for set +%% | term() -- for base types +%% +%% Description: +%%-------------------------------------------------------------------- +write(Proto, {{struct, StructDef}, Data}) + when is_list(StructDef), is_tuple(Data), length(StructDef) == size(Data) - 1 -> + + [StructName | Elems] = tuple_to_list(Data), + ok = write(Proto, #protocol_struct_begin{name = StructName}), + ok = struct_write_loop(Proto, StructDef, Elems), + ok = write(Proto, struct_end), + ok; + +write(Proto, {{struct, {Module, StructureName}}, Data}) + when is_atom(Module), + is_atom(StructureName), + element(1, Data) =:= StructureName -> + StructType = Module:struct_info(StructureName), + write(Proto, {Module:struct_info(StructureName), Data}); + +write(Proto, {{list, Type}, Data}) + when is_list(Data) -> + ok = write(Proto, + #protocol_list_begin{ + etype = term_to_typeid(Type), + size = length(Data) + }), + lists:foreach(fun(Elem) -> + ok = write(Proto, {Type, Elem}) + end, + Data), + ok = write(Proto, list_end), + ok; + +write(Proto, {{map, KeyType, ValType}, Data}) -> + ok = write(Proto, + #protocol_map_begin{ + ktype = term_to_typeid(KeyType), + vtype = term_to_typeid(ValType), + size = dict:size(Data) + }), + dict:fold(fun(KeyData, ValData, _Acc) -> + ok = write(Proto, {KeyType, KeyData}), + ok = write(Proto, {ValType, ValData}) + end, + _AccO = ok, + Data), + ok = write(Proto, map_end), + ok; + +write(Proto, {{set, Type}, Data}) -> + true = sets:is_set(Data), + ok = write(Proto, + #protocol_set_begin{ + etype = term_to_typeid(Type), + size = sets:size(Data) + }), + sets:fold(fun(Elem, _Acc) -> + ok = write(Proto, {Type, Elem}) + end, + _Acc0 = ok, + Data), + ok = write(Proto, set_end), + ok; + +write(#protocol{module = Module, + data = ModuleData}, Data) -> + Module:write(ModuleData, Data). + +struct_write_loop(Proto, [{Fid, Type} | RestStructDef], [Data | RestData]) -> + case Data of + undefined -> + % null fields are skipped in response + skip; + _ -> + ok = write(Proto, + #protocol_field_begin{ + type = term_to_typeid(Type), + id = Fid + }), + ok = write(Proto, {Type, Data}), + ok = write(Proto, field_end) + end, + struct_write_loop(Proto, RestStructDef, RestData); +struct_write_loop(Proto, [], []) -> + ok = write(Proto, field_stop), + ok. diff --git a/lib/erl/src/thrift_server.erl b/lib/erl/src/thrift_server.erl new file mode 100644 index 00000000..5d0012ba --- /dev/null +++ b/lib/erl/src/thrift_server.erl @@ -0,0 +1,183 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_server). + +-behaviour(gen_server). + +%% API +-export([start_link/3, stop/1, take_socket/2]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, {listen_socket, acceptor_ref, service, handler}). + +%%==================================================================== +%% API +%%==================================================================== +%%-------------------------------------------------------------------- +%% Function: start_link() -> {ok,Pid} | ignore | {error,Error} +%% Description: Starts the server +%%-------------------------------------------------------------------- +start_link(Port, Service, HandlerModule) when is_integer(Port), is_atom(HandlerModule) -> + gen_server:start_link({local, ?SERVER}, ?MODULE, {Port, Service, HandlerModule}, []). + +%%-------------------------------------------------------------------- +%% Function: stop(Pid) -> ok, {error, Reason} +%% Description: Stops the server. +%%-------------------------------------------------------------------- +stop(Pid) when is_pid(Pid) -> + gen_server:call(Pid, stop). + + +take_socket(Server, Socket) -> + gen_server:call(Server, {take_socket, Socket}). + + +%%==================================================================== +%% gen_server callbacks +%%==================================================================== + +%%-------------------------------------------------------------------- +%% Function: init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% Description: Initiates the server +%%-------------------------------------------------------------------- +init({Port, Service, Handler}) -> + {ok, Socket} = gen_tcp:listen(Port, + [binary, + {packet, 0}, + {active, false}, + {nodelay, true}, + {reuseaddr, true}]), + {ok, Ref} = prim_inet:async_accept(Socket, -1), + {ok, #state{listen_socket = Socket, + acceptor_ref = Ref, + service = Service, + handler = Handler}}. + +%%-------------------------------------------------------------------- +%% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | +%% {stop, Reason, State} +%% Description: Handling call messages +%%-------------------------------------------------------------------- +handle_call(stop, _From, State) -> + {stop, stopped, ok, State}; + +handle_call({take_socket, Socket}, {FromPid, _Tag}, State) -> + Result = gen_tcp:controlling_process(Socket, FromPid), + {reply, Result, State}. + +%%-------------------------------------------------------------------- +%% Function: handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% Description: Handling cast messages +%%-------------------------------------------------------------------- +handle_cast(_Msg, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% Function: handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% Description: Handling all non call/cast messages +%%-------------------------------------------------------------------- +handle_info({inet_async, ListenSocket, Ref, {ok, ClientSocket}}, + State = #state{listen_socket = ListenSocket, + acceptor_ref = Ref, + service = Service, + handler = Handler}) -> + case set_sockopt(ListenSocket, ClientSocket) of + ok -> + %% New client connected - start processor + start_processor(ClientSocket, Service, Handler), + {ok, NewRef} = prim_inet:async_accept(ListenSocket, -1), + {noreply, State#state{acceptor_ref = NewRef}}; + {error, Reason} -> + error_logger:error_msg("Couldn't set socket opts: ~p~n", + [Reason]), + {stop, Reason, State} + end; + +handle_info({inet_async, ListenSocket, Ref, Error}, State) -> + error_logger:error_msg("Error in acceptor: ~p~n", [Error]), + {stop, Error, State}; + +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% Function: terminate(Reason, State) -> void() +%% Description: This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any necessary +%% cleaning up. When it returns, the gen_server terminates with Reason. +%% The return value is ignored. +%%-------------------------------------------------------------------- +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% Func: code_change(OldVsn, State, Extra) -> {ok, NewState} +%% Description: Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +set_sockopt(ListenSocket, ClientSocket) -> + true = inet_db:register_socket(ClientSocket, inet_tcp), + case prim_inet:getopts(ListenSocket, + [active, nodelay, keepalive, delay_send, priority, tos]) of + {ok, Opts} -> + case prim_inet:setopts(ClientSocket, Opts) of + ok -> ok; + Error -> gen_tcp:close(ClientSocket), + Error + end; + Error -> + gen_tcp:close(ClientSocket), + Error + end. + +start_processor(Socket, Service, Handler) -> + Server = self(), + + ProtoGen = fun() -> + % Become the controlling process + ok = take_socket(Server, Socket), + {ok, SocketTransport} = thrift_socket_transport:new(Socket), + {ok, BufferedTransport} = thrift_buffered_transport:new(SocketTransport), + {ok, Protocol} = thrift_binary_protocol:new(BufferedTransport), + {ok, Protocol, Protocol} + end, + + spawn(thrift_processor, init, [{Server, ProtoGen, Service, Handler}]). diff --git a/lib/erl/src/thrift_service.erl b/lib/erl/src/thrift_service.erl new file mode 100644 index 00000000..2ed7b57b --- /dev/null +++ b/lib/erl/src/thrift_service.erl @@ -0,0 +1,25 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_service). + +-export([behaviour_info/1]). + +behaviour_info(callbacks) -> + [{function_info, 2}]. diff --git a/lib/erl/src/thrift_socket_server.erl b/lib/erl/src/thrift_socket_server.erl new file mode 100644 index 00000000..62bdfdaf --- /dev/null +++ b/lib/erl/src/thrift_socket_server.erl @@ -0,0 +1,249 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_socket_server). + +-behaviour(gen_server). + +-export([start/1, stop/1]). + +-export([init/1, handle_call/3, handle_cast/2, terminate/2, code_change/3, + handle_info/2]). + +-export([acceptor_loop/1]). + +-record(thrift_socket_server, + {port, + service, + handler, + name, + max=2048, + ip=any, + listen=null, + acceptor=null, + socket_opts=[{recv_timeout, 500}] + }). + +start(State=#thrift_socket_server{}) -> + start_server(State); +start(Options) -> + start(parse_options(Options)). + +stop(Name) when is_atom(Name) -> + gen_server:cast(Name, stop); +stop(Pid) when is_pid(Pid) -> + gen_server:cast(Pid, stop); +stop({local, Name}) -> + stop(Name); +stop({global, Name}) -> + stop(Name); +stop(Options) -> + State = parse_options(Options), + stop(State#thrift_socket_server.name). + +%% Internal API + +parse_options(Options) -> + parse_options(Options, #thrift_socket_server{}). + +parse_options([], State) -> + State; +parse_options([{name, L} | Rest], State) when is_list(L) -> + Name = {local, list_to_atom(L)}, + parse_options(Rest, State#thrift_socket_server{name=Name}); +parse_options([{name, A} | Rest], State) when is_atom(A) -> + Name = {local, A}, + parse_options(Rest, State#thrift_socket_server{name=Name}); +parse_options([{name, Name} | Rest], State) -> + parse_options(Rest, State#thrift_socket_server{name=Name}); +parse_options([{port, L} | Rest], State) when is_list(L) -> + Port = list_to_integer(L), + parse_options(Rest, State#thrift_socket_server{port=Port}); +parse_options([{port, Port} | Rest], State) -> + parse_options(Rest, State#thrift_socket_server{port=Port}); +parse_options([{ip, Ip} | Rest], State) -> + ParsedIp = case Ip of + any -> + any; + Ip when is_tuple(Ip) -> + Ip; + Ip when is_list(Ip) -> + {ok, IpTuple} = inet_parse:address(Ip), + IpTuple + end, + parse_options(Rest, State#thrift_socket_server{ip=ParsedIp}); +parse_options([{socket_opts, L} | Rest], State) when is_list(L), length(L) > 0 -> + parse_options(Rest, State#thrift_socket_server{socket_opts=L}); +parse_options([{handler, Handler} | Rest], State) -> + parse_options(Rest, State#thrift_socket_server{handler=Handler}); +parse_options([{service, Service} | Rest], State) -> + parse_options(Rest, State#thrift_socket_server{service=Service}); +parse_options([{max, Max} | Rest], State) -> + MaxInt = case Max of + Max when is_list(Max) -> + list_to_integer(Max); + Max when is_integer(Max) -> + Max + end, + parse_options(Rest, State#thrift_socket_server{max=MaxInt}). + +start_server(State=#thrift_socket_server{name=Name}) -> + case Name of + undefined -> + gen_server:start_link(?MODULE, State, []); + _ -> + gen_server:start_link(Name, ?MODULE, State, []) + end. + +init(State=#thrift_socket_server{ip=Ip, port=Port}) -> + process_flag(trap_exit, true), + BaseOpts = [binary, + {reuseaddr, true}, + {packet, 0}, + {backlog, 4096}, + {recbuf, 8192}, + {active, false}], + Opts = case Ip of + any -> + BaseOpts; + Ip -> + [{ip, Ip} | BaseOpts] + end, + case gen_tcp_listen(Port, Opts, State) of + {stop, eacces} -> + %% fdsrv module allows another shot to bind + %% ports which require root access + case Port < 1024 of + true -> + case fdsrv:start() of + {ok, _} -> + case fdsrv:bind_socket(tcp, Port) of + {ok, Fd} -> + gen_tcp_listen(Port, [{fd, Fd} | Opts], State); + _ -> + {stop, fdsrv_bind_failed} + end; + _ -> + {stop, fdsrv_start_failed} + end; + false -> + {stop, eacces} + end; + Other -> + error_logger:info_msg("thrift service listening on port ~p", [Port]), + Other + end. + +gen_tcp_listen(Port, Opts, State) -> + case gen_tcp:listen(Port, Opts) of + {ok, Listen} -> + {ok, ListenPort} = inet:port(Listen), + {ok, new_acceptor(State#thrift_socket_server{listen=Listen, + port=ListenPort})}; + {error, Reason} -> + {stop, Reason} + end. + +new_acceptor(State=#thrift_socket_server{max=0}) -> + error_logger:error_msg("Not accepting new connections"), + State#thrift_socket_server{acceptor=null}; +new_acceptor(State=#thrift_socket_server{acceptor=OldPid, listen=Listen, + service=Service, handler=Handler, + socket_opts=Opts + }) -> + Pid = proc_lib:spawn_link(?MODULE, acceptor_loop, + [{self(), Listen, Service, Handler, Opts}]), +%% error_logger:info_msg("Spawning new acceptor: ~p => ~p", [OldPid, Pid]), + State#thrift_socket_server{acceptor=Pid}. + +acceptor_loop({Server, Listen, Service, Handler, SocketOpts}) + when is_pid(Server), is_list(SocketOpts) -> + case catch gen_tcp:accept(Listen) of % infinite timeout + {ok, Socket} -> + gen_server:cast(Server, {accepted, self()}), + ProtoGen = fun() -> + {ok, SocketTransport} = thrift_socket_transport:new(Socket, SocketOpts), + {ok, BufferedTransport} = thrift_buffered_transport:new(SocketTransport), + {ok, Protocol} = thrift_binary_protocol:new(BufferedTransport), + {ok, IProt=Protocol, OProt=Protocol} + end, + thrift_processor:init({Server, ProtoGen, Service, Handler}); + {error, closed} -> + exit({error, closed}); + Other -> + error_logger:error_report( + [{application, thrift}, + "Accept failed error", + lists:flatten(io_lib:format("~p", [Other]))]), + exit({error, accept_failed}) + end. + +handle_call({get, port}, _From, State=#thrift_socket_server{port=Port}) -> + {reply, Port, State}; +handle_call(_Message, _From, State) -> + Res = error, + {reply, Res, State}. + +handle_cast({accepted, Pid}, + State=#thrift_socket_server{acceptor=Pid, max=Max}) -> + % io:format("accepted ~p~n", [Pid]), + State1 = State#thrift_socket_server{max=Max - 1}, + {noreply, new_acceptor(State1)}; +handle_cast(stop, State) -> + {stop, normal, State}. + +terminate(_Reason, #thrift_socket_server{listen=Listen, port=Port}) -> + gen_tcp:close(Listen), + case Port < 1024 of + true -> + catch fdsrv:stop(), + ok; + false -> + ok + end. + +code_change(_OldVsn, State, _Extra) -> + State. + +handle_info({'EXIT', Pid, normal}, + State=#thrift_socket_server{acceptor=Pid}) -> + {noreply, new_acceptor(State)}; +handle_info({'EXIT', Pid, Reason}, + State=#thrift_socket_server{acceptor=Pid}) -> + error_logger:error_report({?MODULE, ?LINE, + {acceptor_error, Reason}}), + timer:sleep(100), + {noreply, new_acceptor(State)}; +handle_info({'EXIT', _LoopPid, Reason}, + State=#thrift_socket_server{acceptor=Pid, max=Max}) -> + case Reason of + normal -> ok; + shutdown -> ok; + _ -> error_logger:error_report({?MODULE, ?LINE, + {child_error, Reason, erlang:get_stacktrace()}}) + end, + State1 = State#thrift_socket_server{max=Max + 1}, + State2 = case Pid of + null -> new_acceptor(State1); + _ -> State1 + end, + {noreply, State2}; +handle_info(Info, State) -> + error_logger:info_report([{'INFO', Info}, {'State', State}]), + {noreply, State}. diff --git a/lib/erl/src/thrift_socket_transport.erl b/lib/erl/src/thrift_socket_transport.erl new file mode 100644 index 00000000..fcd69449 --- /dev/null +++ b/lib/erl/src/thrift_socket_transport.erl @@ -0,0 +1,119 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_socket_transport). + +-behaviour(thrift_transport). + +-export([new/1, + new/2, + write/2, read/2, flush/1, close/1, + + new_transport_factory/3]). + +-record(data, {socket, + recv_timeout=infinity}). + +new(Socket) -> + new(Socket, []). + +new(Socket, Opts) when is_list(Opts) -> + State = + case lists:keysearch(recv_timeout, 1, Opts) of + {value, {recv_timeout, Timeout}} + when is_integer(Timeout), Timeout > 0 -> + #data{socket=Socket, recv_timeout=Timeout}; + _ -> + #data{socket=Socket} + end, + thrift_transport:new(?MODULE, State). + +%% Data :: iolist() +write(#data{socket = Socket}, Data) -> + gen_tcp:send(Socket, Data). + +read(#data{socket=Socket, recv_timeout=Timeout}, Len) + when is_integer(Len), Len >= 0 -> + case gen_tcp:recv(Socket, Len, Timeout) of + Err = {error, timeout} -> + error_logger:info_msg("read timeout: peer conn ~p", [inet:peername(Socket)]), + gen_tcp:close(Socket), + Err; + Data -> Data + end. + +%% We can't really flush - everything is flushed when we write +flush(_) -> + ok. + +close(#data{socket = Socket}) -> + gen_tcp:close(Socket). + + +%%%% FACTORY GENERATION %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +%% The following "local" record is filled in by parse_factory_options/2 +%% below. These options can be passed to new_protocol_factory/3 in a +%% proplists-style option list. They're parsed like this so it is an O(n) +%% operation instead of O(n^2) +-record(factory_opts, {connect_timeout = infinity, + sockopts = [], + framed = false}). + +parse_factory_options([], Opts) -> + Opts; +parse_factory_options([{framed, Bool} | Rest], Opts) when is_boolean(Bool) -> + parse_factory_options(Rest, Opts#factory_opts{framed=Bool}); +parse_factory_options([{sockopts, OptList} | Rest], Opts) when is_list(OptList) -> + parse_factory_options(Rest, Opts#factory_opts{sockopts=OptList}); +parse_factory_options([{connect_timeout, TO} | Rest], Opts) when TO =:= infinity; is_integer(TO) -> + parse_factory_options(Rest, Opts#factory_opts{connect_timeout=TO}). + + +%% +%% Generates a "transport factory" function - a fun which returns a thrift_transport() +%% instance. +%% This can be passed into a protocol factory to generate a connection to a +%% thrift server over a socket. +%% +new_transport_factory(Host, Port, Options) -> + ParsedOpts = parse_factory_options(Options, #factory_opts{}), + + F = fun() -> + SockOpts = [binary, + {packet, 0}, + {active, false}, + {nodelay, true} | + ParsedOpts#factory_opts.sockopts], + case catch gen_tcp:connect(Host, Port, SockOpts, + ParsedOpts#factory_opts.connect_timeout) of + {ok, Sock} -> + {ok, Transport} = thrift_socket_transport:new(Sock), + {ok, BufTransport} = + case ParsedOpts#factory_opts.framed of + true -> thrift_framed_transport:new(Transport); + false -> thrift_buffered_transport:new(Transport) + end, + {ok, BufTransport}; + Error -> + Error + end + end, + {ok, F}. diff --git a/lib/erl/src/thrift_transport.erl b/lib/erl/src/thrift_transport.erl new file mode 100644 index 00000000..20c4b5dc --- /dev/null +++ b/lib/erl/src/thrift_transport.erl @@ -0,0 +1,57 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(thrift_transport). + +-export([behaviour_info/1]). + +-export([new/2, + write/2, + read/2, + flush/1, + close/1 + ]). + +behaviour_info(callbacks) -> + [{read, 2}, + {write, 2}, + {flush, 1}, + {close, 1} + ]. + +-record(transport, {module, data}). + +new(Module, Data) when is_atom(Module) -> + {ok, #transport{module = Module, + data = Data}}. + +%% Data :: iolist() +write(Transport, Data) -> + Module = Transport#transport.module, + Module:write(Transport#transport.data, Data). + +read(Transport, Len) when is_integer(Len) -> + Module = Transport#transport.module, + Module:read(Transport#transport.data, Len). + +flush(#transport{module = Module, data = Data}) -> + Module:flush(Data). + +close(#transport{module = Module, data = Data}) -> + Module:close(Data). diff --git a/lib/erl/vsn.mk b/lib/erl/vsn.mk new file mode 100644 index 00000000..d9b40014 --- /dev/null +++ b/lib/erl/vsn.mk @@ -0,0 +1 @@ +THRIFT_VSN=0.1 diff --git a/lib/hs/.gitignore b/lib/hs/.gitignore new file mode 100644 index 00000000..849ddff3 --- /dev/null +++ b/lib/hs/.gitignore @@ -0,0 +1 @@ +dist/ diff --git a/lib/hs/README b/lib/hs/README new file mode 100644 index 00000000..e58c8c93 --- /dev/null +++ b/lib/hs/README @@ -0,0 +1,82 @@ +Haskell Thrift Bindings + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Running +======= + +You need -fglasgow-exts. Use Cabal to compile and install. If you're trying to +manually compile or load via ghci, and you're using ghc 6.10 (or really if your +default base package has major version number 4), you must specify a version of +the base package with major version number 3. Furthermore if you have the syb +package installed you need to hide that package to avoid import conflicts. +Here's an example of what I'm talking about: + + ghci -fglasgow-exts -package base-3.0.3.0 -hide-package syb -isrc Thrift.hs + +To determine which versions of the base package you have installed use the +following command: + + ghc-pkg list base + +All of this is taken care of for you if you use Cabal. + + +Enums +===== + +become haskell data types. Use fromEnum to get out the int value. + +Structs +======= + +become records. Field labels are ugly, of the form f_STRUCTNAME_FIELDNAME. All +fields are Maybe types. + +Exceptions +========== + +identical to structs. Throw them with throwDyn. Catch them with catchDyn. + +Client +====== + +just a bunch of functions. You may have to import a bunch of client files to +deal with inheritance. + +Interface +========= + +You should only have to import the last one in the chain of inheritors. To make +an interface, declare a label: + + data MyIface = MyIface + +and then declare it an instance of each iface class, starting with the superest +class and proceding down (all the while defining the methods). Then pass your +label to process as the handler. + +Processor +========= + +Just a function that takes a handler label, protocols. It calls the +superclasses process if there is a superclass. + diff --git a/lib/hs/Setup.lhs b/lib/hs/Setup.lhs new file mode 100644 index 00000000..c9e6d970 --- /dev/null +++ b/lib/hs/Setup.lhs @@ -0,0 +1,23 @@ +#!/usr/bin/env runhaskell + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +> import Distribution.Simple +> main = defaultMain diff --git a/lib/hs/TODO b/lib/hs/TODO new file mode 100644 index 00000000..13681732 --- /dev/null +++ b/lib/hs/TODO @@ -0,0 +1,2 @@ +The library could stand to be built up more. +Many modules need export lists. diff --git a/lib/hs/Thrift.cabal b/lib/hs/Thrift.cabal new file mode 100644 index 00000000..4cef4de6 --- /dev/null +++ b/lib/hs/Thrift.cabal @@ -0,0 +1,20 @@ +Name: Thrift +Version: 0.1.0 +Cabal-Version: >= 1.2 +License: Apache2 +Category: Foreign +Build-Type: Simple +Synopsis: Thrift library package + +Library + Hs-Source-Dirs: + src + Build-Depends: + base >=4, network, ghc-prim + ghc-options: + -fglasgow-exts + Extensions: + DeriveDataTypeable + Exposed-Modules: + Thrift, Thrift.Protocol, Thrift.Transport, Thrift.Protocol.Binary + Thrift.Transport.Handle, Thrift.Server diff --git a/lib/hs/src/Thrift.hs b/lib/hs/src/Thrift.hs new file mode 100644 index 00000000..291bcae5 --- /dev/null +++ b/lib/hs/src/Thrift.hs @@ -0,0 +1,111 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +module Thrift + ( module Thrift.Transport + , module Thrift.Protocol + , AppExnType(..) + , AppExn(..) + , readAppExn + , writeAppExn + , ThriftException(..) + ) where + +import Control.Monad ( when ) +import Control.Exception + +import Data.Typeable ( Typeable ) + +import Thrift.Transport +import Thrift.Protocol + + +data ThriftException = ThriftException + deriving ( Show, Typeable ) +instance Exception ThriftException + +data AppExnType + = AE_UNKNOWN + | AE_UNKNOWN_METHOD + | AE_INVALID_MESSAGE_TYPE + | AE_WRONG_METHOD_NAME + | AE_BAD_SEQUENCE_ID + | AE_MISSING_RESULT + deriving ( Eq, Show, Typeable ) + +instance Enum AppExnType where + toEnum 0 = AE_UNKNOWN + toEnum 1 = AE_UNKNOWN_METHOD + toEnum 2 = AE_INVALID_MESSAGE_TYPE + toEnum 3 = AE_WRONG_METHOD_NAME + toEnum 4 = AE_BAD_SEQUENCE_ID + toEnum 5 = AE_MISSING_RESULT + + fromEnum AE_UNKNOWN = 0 + fromEnum AE_UNKNOWN_METHOD = 1 + fromEnum AE_INVALID_MESSAGE_TYPE = 2 + fromEnum AE_WRONG_METHOD_NAME = 3 + fromEnum AE_BAD_SEQUENCE_ID = 4 + fromEnum AE_MISSING_RESULT = 5 + +data AppExn = AppExn { ae_type :: AppExnType, ae_message :: String } + deriving ( Show, Typeable ) +instance Exception AppExn + +writeAppExn :: (Protocol p, Transport t) => p t -> AppExn -> IO () +writeAppExn pt ae = do + writeStructBegin pt "TApplicationException" + + when (ae_message ae /= "") $ do + writeFieldBegin pt ("message", T_STRING , 1) + writeString pt (ae_message ae) + writeFieldEnd pt + + writeFieldBegin pt ("type", T_I32, 2); + writeI32 pt (fromEnum (ae_type ae)) + writeFieldEnd pt + writeFieldStop pt + writeStructEnd pt + +readAppExn :: (Protocol p, Transport t) => p t -> IO AppExn +readAppExn pt = do + readStructBegin pt + rec <- readAppExnFields pt (AppExn {ae_type = undefined, ae_message = undefined}) + readStructEnd pt + return rec + +readAppExnFields pt rec = do + (n, ft, id) <- readFieldBegin pt + if ft == T_STOP + then return rec + else case id of + 1 -> if ft == T_STRING then + do s <- readString pt + readAppExnFields pt rec{ae_message = s} + else do skip pt ft + readAppExnFields pt rec + 2 -> if ft == T_I32 then + do i <- readI32 pt + readAppExnFields pt rec{ae_type = (toEnum i)} + else do skip pt ft + readAppExnFields pt rec + _ -> do skip pt ft + readFieldEnd pt + readAppExnFields pt rec + diff --git a/lib/hs/src/Thrift/Protocol.hs b/lib/hs/src/Thrift/Protocol.hs new file mode 100644 index 00000000..8fa060ea --- /dev/null +++ b/lib/hs/src/Thrift/Protocol.hs @@ -0,0 +1,191 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +module Thrift.Protocol + ( Protocol(..) + , skip + , MessageType(..) + , ThriftType(..) + , ProtocolExn(..) + , ProtocolExnType(..) + ) where + +import Control.Monad ( replicateM_, unless ) +import Control.Exception + +import Data.Typeable ( Typeable ) +import Data.Int + +import Thrift.Transport + + +data ThriftType + = T_STOP + | T_VOID + | T_BOOL + | T_BYTE + | T_DOUBLE + | T_I16 + | T_I32 + | T_I64 + | T_STRING + | T_STRUCT + | T_MAP + | T_SET + | T_LIST + deriving ( Eq ) + +instance Enum ThriftType where + fromEnum T_STOP = 0 + fromEnum T_VOID = 1 + fromEnum T_BOOL = 2 + fromEnum T_BYTE = 3 + fromEnum T_DOUBLE = 4 + fromEnum T_I16 = 6 + fromEnum T_I32 = 8 + fromEnum T_I64 = 10 + fromEnum T_STRING = 11 + fromEnum T_STRUCT = 12 + fromEnum T_MAP = 13 + fromEnum T_SET = 14 + fromEnum T_LIST = 15 + + toEnum 0 = T_STOP + toEnum 1 = T_VOID + toEnum 2 = T_BOOL + toEnum 3 = T_BYTE + toEnum 4 = T_DOUBLE + toEnum 6 = T_I16 + toEnum 8 = T_I32 + toEnum 10 = T_I64 + toEnum 11 = T_STRING + toEnum 12 = T_STRUCT + toEnum 13 = T_MAP + toEnum 14 = T_SET + toEnum 15 = T_LIST + +data MessageType + = M_CALL + | M_REPLY + | M_EXCEPTION + deriving ( Eq ) + +instance Enum MessageType where + fromEnum M_CALL = 1 + fromEnum M_REPLY = 2 + fromEnum M_EXCEPTION = 3 + + toEnum 1 = M_CALL + toEnum 2 = M_REPLY + toEnum 3 = M_EXCEPTION + + +class Protocol a where + getTransport :: Transport t => a t -> t + + writeMessageBegin :: Transport t => a t -> (String, MessageType, Int) -> IO () + writeMessageEnd :: Transport t => a t -> IO () + + writeStructBegin :: Transport t => a t -> String -> IO () + writeStructEnd :: Transport t => a t -> IO () + writeFieldBegin :: Transport t => a t -> (String, ThriftType, Int) -> IO () + writeFieldEnd :: Transport t => a t -> IO () + writeFieldStop :: Transport t => a t -> IO () + writeMapBegin :: Transport t => a t -> (ThriftType, ThriftType, Int) -> IO () + writeMapEnd :: Transport t => a t -> IO () + writeListBegin :: Transport t => a t -> (ThriftType, Int) -> IO () + writeListEnd :: Transport t => a t -> IO () + writeSetBegin :: Transport t => a t -> (ThriftType, Int) -> IO () + writeSetEnd :: Transport t => a t -> IO () + + writeBool :: Transport t => a t -> Bool -> IO () + writeByte :: Transport t => a t -> Int -> IO () + writeI16 :: Transport t => a t -> Int -> IO () + writeI32 :: Transport t => a t -> Int -> IO () + writeI64 :: Transport t => a t -> Int64 -> IO () + writeDouble :: Transport t => a t -> Double -> IO () + writeString :: Transport t => a t -> String -> IO () + writeBinary :: Transport t => a t -> String -> IO () + + + readMessageBegin :: Transport t => a t -> IO (String, MessageType, Int) + readMessageEnd :: Transport t => a t -> IO () + + readStructBegin :: Transport t => a t -> IO String + readStructEnd :: Transport t => a t -> IO () + readFieldBegin :: Transport t => a t -> IO (String, ThriftType, Int) + readFieldEnd :: Transport t => a t -> IO () + readMapBegin :: Transport t => a t -> IO (ThriftType, ThriftType, Int) + readMapEnd :: Transport t => a t -> IO () + readListBegin :: Transport t => a t -> IO (ThriftType, Int) + readListEnd :: Transport t => a t -> IO () + readSetBegin :: Transport t => a t -> IO (ThriftType, Int) + readSetEnd :: Transport t => a t -> IO () + + readBool :: Transport t => a t -> IO Bool + readByte :: Transport t => a t -> IO Int + readI16 :: Transport t => a t -> IO Int + readI32 :: Transport t => a t -> IO Int + readI64 :: Transport t => a t -> IO Int64 + readDouble :: Transport t => a t -> IO Double + readString :: Transport t => a t -> IO String + readBinary :: Transport t => a t -> IO String + + +skip :: (Protocol p, Transport t) => p t -> ThriftType -> IO () +skip p T_STOP = return () +skip p T_VOID = return () +skip p T_BOOL = readBool p >> return () +skip p T_BYTE = readByte p >> return () +skip p T_I16 = readI16 p >> return () +skip p T_I32 = readI32 p >> return () +skip p T_I64 = readI64 p >> return () +skip p T_DOUBLE = readDouble p >> return () +skip p T_STRING = readString p >> return () +skip p T_STRUCT = do readStructBegin p + skipFields p + readStructEnd p +skip p T_MAP = do (k, v, s) <- readMapBegin p + replicateM_ s (skip p k >> skip p v) + readMapEnd p +skip p T_SET = do (t, n) <- readSetBegin p + replicateM_ n (skip p t) + readSetEnd p +skip p T_LIST = do (t, n) <- readListBegin p + replicateM_ n (skip p t) + readListEnd p + + +skipFields :: (Protocol p, Transport t) => p t -> IO () +skipFields p = do + (_, t, _) <- readFieldBegin p + unless (t == T_STOP) (skip p t >> readFieldEnd p >> skipFields p) + + +data ProtocolExnType + = PE_UNKNOWN + | PE_INVALID_DATA + | PE_NEGATIVE_SIZE + | PE_SIZE_LIMIT + | PE_BAD_VERSION + deriving ( Eq, Show, Typeable ) + +data ProtocolExn = ProtocolExn ProtocolExnType String + deriving ( Show, Typeable ) +instance Exception ProtocolExn diff --git a/lib/hs/src/Thrift/Protocol/Binary.hs b/lib/hs/src/Thrift/Protocol/Binary.hs new file mode 100644 index 00000000..3f798cee --- /dev/null +++ b/lib/hs/src/Thrift/Protocol/Binary.hs @@ -0,0 +1,147 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +module Thrift.Protocol.Binary + ( module Thrift.Protocol + , BinaryProtocol(..) + ) where + +import Control.Exception ( throw ) + +import Data.Bits +import Data.Int +import Data.List ( foldl' ) + +import GHC.Exts +import GHC.Word + +import Thrift.Protocol +import Thrift.Transport + + +version_mask = 0xffff0000 +version_1 = 0x80010000 + +data BinaryProtocol a = Transport a => BinaryProtocol a + + +instance Protocol BinaryProtocol where + getTransport (BinaryProtocol t) = t + + writeMessageBegin p (n, t, s) = do + writeI32 p (version_1 .|. (fromEnum t)) + writeString p n + writeI32 p s + writeMessageEnd _ = return () + + writeStructBegin _ _ = return () + writeStructEnd _ = return () + writeFieldBegin p (_, t, i) = writeType p t >> writeI16 p i + writeFieldEnd _ = return () + writeFieldStop p = writeType p T_STOP + writeMapBegin p (k, v, n) = writeType p k >> writeType p v >> writeI32 p n + writeMapEnd p = return () + writeListBegin p (t, n) = writeType p t >> writeI32 p n + writeListEnd _ = return () + writeSetBegin p (t, n) = writeType p t >> writeI32 p n + writeSetEnd _ = return () + + writeBool p b = tWrite (getTransport p) [toEnum $ if b then 1 else 0] + writeByte p b = tWrite (getTransport p) (getBytes b 1) + writeI16 p b = tWrite (getTransport p) (getBytes b 2) + writeI32 p b = tWrite (getTransport p) (getBytes b 4) + writeI64 p b = tWrite (getTransport p) (getBytes b 8) + writeDouble p d = writeI64 p (fromIntegral $ floatBits d) + writeString p s = writeI32 p (length s) >> tWrite (getTransport p) s + writeBinary = writeString + + readMessageBegin p = do + ver <- readI32 p + if (ver .&. version_mask /= version_1) + then throw $ ProtocolExn PE_BAD_VERSION "Missing version identifier" + else do + s <- readString p + sz <- readI32 p + return (s, toEnum $ ver .&. 0xFF, sz) + readMessageEnd _ = return () + readStructBegin _ = return "" + readStructEnd _ = return () + readFieldBegin p = do + t <- readType p + n <- if t /= T_STOP then readI16 p else return 0 + return ("", t, n) + readFieldEnd _ = return () + readMapBegin p = do + kt <- readType p + vt <- readType p + n <- readI32 p + return (kt, vt, n) + readMapEnd _ = return () + readListBegin p = do + t <- readType p + n <- readI32 p + return (t, n) + readListEnd _ = return () + readSetBegin p = do + t <- readType p + n <- readI32 p + return (t, n) + readSetEnd _ = return () + + readBool p = (== 1) `fmap` readByte p + readByte p = do + bs <- tReadAll (getTransport p) 1 + return $ fromIntegral (composeBytes bs :: Int8) + readI16 p = do + bs <- tReadAll (getTransport p) 2 + return $ fromIntegral (composeBytes bs :: Int16) + readI32 p = composeBytes `fmap` tReadAll (getTransport p) 4 + readI64 p = composeBytes `fmap` tReadAll (getTransport p) 8 + readDouble p = do + bs <- readI64 p + return $ floatOfBits $ fromIntegral bs + readString p = readI32 p >>= tReadAll (getTransport p) + readBinary = readString + + +-- | Write a type as a byte +writeType :: (Protocol p, Transport t) => p t -> ThriftType -> IO () +writeType p t = writeByte p (fromEnum t) + +-- | Read a byte as though it were a ThriftType +readType :: (Protocol p, Transport t) => p t -> IO ThriftType +readType p = toEnum `fmap` readByte p + +composeBytes :: (Bits b, Enum t) => [t] -> b +composeBytes = (foldl' fn 0) . (map $ fromIntegral . fromEnum) + where fn acc b = (acc `shiftL` 8) .|. b + +getByte :: Bits a => a -> Int -> a +getByte i n = 255 .&. (i `shiftR` (8 * n)) + +getBytes :: (Bits a, Integral a) => a -> Int -> String +getBytes i 0 = [] +getBytes i n = (toEnum $ fromIntegral $ getByte i (n-1)):(getBytes i (n-1)) + +floatBits :: Double -> Word64 +floatBits (D# d#) = W64# (unsafeCoerce# d#) + +floatOfBits :: Word64 -> Double +floatOfBits (W64# b#) = D# (unsafeCoerce# b#) + diff --git a/lib/hs/src/Thrift/Server.hs b/lib/hs/src/Thrift/Server.hs new file mode 100644 index 00000000..770965f1 --- /dev/null +++ b/lib/hs/src/Thrift/Server.hs @@ -0,0 +1,65 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +module Thrift.Server + ( runBasicServer + , runThreadedServer + ) where + +import Control.Concurrent ( forkIO ) +import Control.Exception +import Control.Monad ( forever, when ) + +import Network + +import System.IO + +import Thrift +import Thrift.Transport.Handle +import Thrift.Protocol.Binary + + +-- | A threaded sever that is capable of using any Transport or Protocol +-- instances. +runThreadedServer :: (Transport t, Protocol i, Protocol o) + => (Socket -> IO (i t, o t)) + -> h + -> (h -> (i t, o t) -> IO Bool) + -> PortID + -> IO a +runThreadedServer accepter hand proc port = do + socket <- listenOn port + acceptLoop (accepter socket) (proc hand) + +-- | A basic threaded binary protocol socket server. +runBasicServer :: h + -> (h -> (BinaryProtocol Handle, BinaryProtocol Handle) -> IO Bool) + -> PortNumber + -> IO a +runBasicServer hand proc port = runThreadedServer binaryAccept hand proc (PortNumber port) + where binaryAccept s = do + (h, _, _) <- accept s + return (BinaryProtocol h, BinaryProtocol h) + +acceptLoop :: IO t -> (t -> IO Bool) -> IO a +acceptLoop accepter proc = forever $ + do ps <- accepter + forkIO $ handle (\(e :: SomeException) -> return ()) + (loop $ proc ps) + where loop m = do { continue <- m; when continue (loop m) } diff --git a/lib/hs/src/Thrift/Transport.hs b/lib/hs/src/Thrift/Transport.hs new file mode 100644 index 00000000..29f50d07 --- /dev/null +++ b/lib/hs/src/Thrift/Transport.hs @@ -0,0 +1,60 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +module Thrift.Transport + ( Transport(..) + , TransportExn(..) + , TransportExnType(..) + ) where + +import Control.Monad ( when ) +import Control.Exception ( Exception, throw ) + +import Data.Typeable ( Typeable ) + + +class Transport a where + tIsOpen :: a -> IO Bool + tClose :: a -> IO () + tRead :: a -> Int -> IO String + tWrite :: a -> String ->IO () + tFlush :: a -> IO () + tReadAll :: a -> Int -> IO String + + tReadAll a 0 = return [] + tReadAll a len = do + result <- tRead a len + let rlen = length result + when (rlen == 0) (throw $ TransportExn "Cannot read. Remote side has closed." TE_UNKNOWN) + if len <= rlen + then return result + else (result ++) `fmap` (tReadAll a (len - rlen)) + +data TransportExn = TransportExn String TransportExnType + deriving ( Show, Typeable ) +instance Exception TransportExn + +data TransportExnType + = TE_UNKNOWN + | TE_NOT_OPEN + | TE_ALREADY_OPEN + | TE_TIMED_OUT + | TE_END_OF_FILE + deriving ( Eq, Show, Typeable ) + diff --git a/lib/hs/src/Thrift/Transport/Handle.hs b/lib/hs/src/Thrift/Transport/Handle.hs new file mode 100644 index 00000000..e49456b5 --- /dev/null +++ b/lib/hs/src/Thrift/Transport/Handle.hs @@ -0,0 +1,58 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +module Thrift.Transport.Handle + ( module Thrift.Transport + , HandleSource(..) + ) where + +import Control.Exception ( throw ) +import Control.Monad ( replicateM ) + +import Network + +import System.IO +import System.IO.Error ( isEOFError ) + +import Thrift.Transport + + +instance Transport Handle where + tIsOpen = hIsOpen + tClose h = hClose h + tRead h n = replicateM n (hGetChar h) `catch` handleEOF + tWrite h s = mapM_ (hPutChar h) s + tFlush = hFlush + + +-- | Type class for all types that can open a Handle. This class is used to +-- replace tOpen in the Transport type class. +class HandleSource s where + hOpen :: s -> IO Handle + +instance HandleSource FilePath where + hOpen s = openFile s ReadWriteMode + +instance HandleSource (HostName, PortID) where + hOpen = uncurry connectTo + + +handleEOF e = if isEOFError e + then return [] + else throw $ TransportExn "TChannelTransport: Could not read" TE_UNKNOWN diff --git a/lib/java/Makefile.am b/lib/java/Makefile.am new file mode 100644 index 00000000..0a40496d --- /dev/null +++ b/lib/java/Makefile.am @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +EXTRA_DIST = build.xml ivy.xml src test + +all-local: + $(ANT) + +install-exec-hook: + $(ANT) install -Dinstall.path=$(DESTDIR)$(JAVA_PREFIX) \ + -Dinstall.javadoc.path=$(DESTDIR)$(docdir)/java + +# Make sure this doesn't fail if ant is not configured. +clean-local: + ANT=$(ANT) ; if test -z "$$ANT" ; then ANT=: ; fi ; \ + $$ANT clean + +check-local: all + $(ANT) test + diff --git a/lib/java/README b/lib/java/README new file mode 100644 index 00000000..6b8d351b --- /dev/null +++ b/lib/java/README @@ -0,0 +1,43 @@ +Thrift Java Software Library + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Using Thrift with Java +====================== + +The Thrift Java source is not build using the GNU tools, but rather uses +the Apache Ant build system, which tends to be predominant amongst Java +developers. + +To compile the Java Thrift libraries, simply do the following: + +ant + +Yep, that's easy. Look for libthrift.jar in the base directory. + +To include Thrift in your applications simply add libthrift.jar to your +classpath, or install if in your default system classpath of choice. + +Dependencies +============ + +Apache Ant +http://ant.apache.org/ diff --git a/lib/java/build.xml b/lib/java/build.xml new file mode 100644 index 00000000..0a7c8944 --- /dev/null +++ b/lib/java/build.xml @@ -0,0 +1,192 @@ + + + + + Thrift Build File + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + You need Apache Ivy 2.0 or later from http://ant.apache.org/ + It could not be loaded from ${ivy_repo_url} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/java/ivy.xml b/lib/java/ivy.xml new file mode 100644 index 00000000..0b1be5d8 --- /dev/null +++ b/lib/java/ivy.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/lib/java/src/org/apache/thrift/IntRangeSet.java b/lib/java/src/org/apache/thrift/IntRangeSet.java new file mode 100644 index 00000000..5430134d --- /dev/null +++ b/lib/java/src/org/apache/thrift/IntRangeSet.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * IntRangeSet is a specialized Set implementation designed + * specifically to make the generated validate() method calls faster. It groups + * the set values into ranges, and in the contains() call, it does + * num ranges * 2 comparisons max. For the common case, which is a single, + * contiguous range, this approach is about 60% faster than using a HashSet. If + * you had a very ragged value set, like all the odd numbers, for instance, + * then you would end up with pretty poor running time. + */ +public class IntRangeSet implements Set { + /** + * This array keeps the bounds of each extent in alternating cells, always + * increasing. Example: [0,5,10,15], which corresponds to 0-5, 10-15. + */ + private int[] extents; + + /** + * We'll keep a duplicate, real HashSet around internally to satisfy some of + * the other set operations. + */ + private Set realSet = new HashSet(); + + public IntRangeSet(int... values) { + Arrays.sort(values); + + List extent_list = new ArrayList(); + + int ext_start = values[0]; + int ext_end_so_far = values[0]; + for (int i = 1; i < values.length; i++) { + realSet.add(values[i]); + + if (values[i] == ext_end_so_far + 1) { + // advance the end so far + ext_end_so_far = values[i]; + } else { + // create an extent for everything we saw so far, move on to the next one + extent_list.add(ext_start); + extent_list.add(ext_end_so_far); + ext_start = values[i]; + ext_end_so_far = values[i]; + } + } + extent_list.add(ext_start); + extent_list.add(ext_end_so_far); + + extents = new int[extent_list.size()]; + for (int i = 0; i < extent_list.size(); i++) { + extents[i] = extent_list.get(i); + } + } + + public boolean add(Integer i) { + throw new UnsupportedOperationException(); + } + + public void clear() { + throw new UnsupportedOperationException(); + } + + public boolean addAll(Collection arg0) { + throw new UnsupportedOperationException(); + } + + /** + * While this method is here for Set interface compatibility, you should avoid + * using it. It incurs boxing overhead! Use the int method directly, instead. + */ + public boolean contains(Object arg0) { + return contains(((Integer)arg0).intValue()); + } + + /** + * This is much faster, since it doesn't stop at Integer on the way through. + * @param val the value you want to check set membership for + * @return true if val was found, false otherwise + */ + public boolean contains(int val) { + for (int i = 0; i < extents.length / 2; i++) { + if (val < extents[i*2]) { + return false; + } else if (val <= extents[i*2+1]) { + return true; + } + } + + return false; + } + + public boolean containsAll(Collection arg0) { + for (Object o : arg0) { + if (!contains(o)) { + return false; + } + } + return true; + } + + public boolean isEmpty() { + return realSet.isEmpty(); + } + + public Iterator iterator() { + return realSet.iterator(); + } + + public boolean remove(Object arg0) { + throw new UnsupportedOperationException(); + } + + public boolean removeAll(Collection arg0) { + throw new UnsupportedOperationException(); + } + + public boolean retainAll(Collection arg0) { + throw new UnsupportedOperationException(); + } + + public int size() { + return realSet.size(); + } + + public Object[] toArray() { + return realSet.toArray(); + } + + public T[] toArray(T[] arg0) { + return realSet.toArray(arg0); + } + + @Override + public String toString() { + String buf = ""; + for (int i = 0; i < extents.length / 2; i++) { + if (i != 0) { + buf += ", "; + } + buf += "[" + extents[i*2] + "," + extents[i*2+1] + "]"; + } + return buf; + } +} diff --git a/lib/java/src/org/apache/thrift/TApplicationException.java b/lib/java/src/org/apache/thrift/TApplicationException.java new file mode 100644 index 00000000..a85e3705 --- /dev/null +++ b/lib/java/src/org/apache/thrift/TApplicationException.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import org.apache.thrift.protocol.TField; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolUtil; +import org.apache.thrift.protocol.TStruct; +import org.apache.thrift.protocol.TType; + +/** + * Application level exception + * + */ +public class TApplicationException extends TException { + + private static final TStruct TAPPLICATION_EXCEPTION_STRUCT = new TStruct("TApplicationException"); + private static final TField MESSAGE_FIELD = new TField("message", TType.STRING, (short)1); + private static final TField TYPE_FIELD = new TField("type", TType.I32, (short)2); + + private static final long serialVersionUID = 1L; + + public static final int UNKNOWN = 0; + public static final int UNKNOWN_METHOD = 1; + public static final int INVALID_MESSAGE_TYPE = 2; + public static final int WRONG_METHOD_NAME = 3; + public static final int BAD_SEQUENCE_ID = 4; + public static final int MISSING_RESULT = 5; + + protected int type_ = UNKNOWN; + + public TApplicationException() { + super(); + } + + public TApplicationException(int type) { + super(); + type_ = type; + } + + public TApplicationException(int type, String message) { + super(message); + type_ = type; + } + + public TApplicationException(String message) { + super(message); + } + + public int getType() { + return type_; + } + + public static TApplicationException read(TProtocol iprot) throws TException { + TField field; + iprot.readStructBegin(); + + String message = null; + int type = UNKNOWN; + + while (true) { + field = iprot.readFieldBegin(); + if (field.type == TType.STOP) { + break; + } + switch (field.id) { + case 1: + if (field.type == TType.STRING) { + message = iprot.readString(); + } else { + TProtocolUtil.skip(iprot, field.type); + } + break; + case 2: + if (field.type == TType.I32) { + type = iprot.readI32(); + } else { + TProtocolUtil.skip(iprot, field.type); + } + break; + default: + TProtocolUtil.skip(iprot, field.type); + break; + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + return new TApplicationException(type, message); + } + + public void write(TProtocol oprot) throws TException { + oprot.writeStructBegin(TAPPLICATION_EXCEPTION_STRUCT); + if (getMessage() != null) { + oprot.writeFieldBegin(MESSAGE_FIELD); + oprot.writeString(getMessage()); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TYPE_FIELD); + oprot.writeI32(type_); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } +} diff --git a/lib/java/src/org/apache/thrift/TBase.java b/lib/java/src/org/apache/thrift/TBase.java new file mode 100644 index 00000000..7c8978a2 --- /dev/null +++ b/lib/java/src/org/apache/thrift/TBase.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import org.apache.thrift.protocol.TProtocol; + +/** + * Generic base interface for generated Thrift objects. + * + */ +public interface TBase extends Cloneable { + + /** + * Reads the TObject from the given input protocol. + * + * @param iprot Input protocol + */ + public void read(TProtocol iprot) throws TException; + + /** + * Writes the objects out to the protocol + * + * @param oprot Output protocol + */ + public void write(TProtocol oprot) throws TException; + + /** + * Check if a field is currently set or unset. + * + * @param fieldId The field's id tag as found in the IDL. + */ + public boolean isSet(int fieldId); + + /** + * Get a field's value by id. Primitive types will be wrapped in the + * appropriate "boxed" types. + * + * @param fieldId The field's id tag as found in the IDL. + */ + public Object getFieldValue(int fieldId); + + /** + * Set a field's value by id. Primitive types must be "boxed" in the + * appropriate object wrapper type. + * + * @param fieldId The field's id tag as found in the IDL. + */ + public void setFieldValue(int fieldId, Object value); +} diff --git a/lib/java/src/org/apache/thrift/TByteArrayOutputStream.java b/lib/java/src/org/apache/thrift/TByteArrayOutputStream.java new file mode 100644 index 00000000..e35fbcb7 --- /dev/null +++ b/lib/java/src/org/apache/thrift/TByteArrayOutputStream.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import java.io.ByteArrayOutputStream; + +/** + * Class that allows access to the underlying buf without doing deep + * copies on it. + * + */ +public class TByteArrayOutputStream extends ByteArrayOutputStream { + public TByteArrayOutputStream(int size) { + super(size); + } + + public TByteArrayOutputStream() { + super(); + } + + + public byte[] get() { + return buf; + } + + public int len() { + return count; + } +} diff --git a/lib/java/src/org/apache/thrift/TDeserializer.java b/lib/java/src/org/apache/thrift/TDeserializer.java new file mode 100644 index 00000000..d6dd5d4b --- /dev/null +++ b/lib/java/src/org/apache/thrift/TDeserializer.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import java.io.ByteArrayInputStream; +import java.io.UnsupportedEncodingException; + +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TIOStreamTransport; + +/** + * Generic utility for easily deserializing objects from a byte array or Java + * String. + * + */ +public class TDeserializer { + private final TProtocolFactory protocolFactory_; + + /** + * Create a new TDeserializer that uses the TBinaryProtocol by default. + */ + public TDeserializer() { + this(new TBinaryProtocol.Factory()); + } + + /** + * Create a new TDeserializer. It will use the TProtocol specified by the + * factory that is passed in. + * + * @param protocolFactory Factory to create a protocol + */ + public TDeserializer(TProtocolFactory protocolFactory) { + protocolFactory_ = protocolFactory; + } + + /** + * Deserialize the Thrift object from a byte array. + * + * @param base The object to read into + * @param bytes The array to read from + */ + public void deserialize(TBase base, byte[] bytes) throws TException { + base.read( + protocolFactory_.getProtocol( + new TIOStreamTransport( + new ByteArrayInputStream(bytes)))); + } + + /** + * Deserialize the Thrift object from a Java string, using a specified + * character set for decoding. + * + * @param base The object to read into + * @param data The string to read from + * @param charset Valid JVM charset + */ + public void deserialize(TBase base, String data, String charset) throws TException { + try { + deserialize(base, data.getBytes(charset)); + } catch (UnsupportedEncodingException uex) { + throw new TException("JVM DOES NOT SUPPORT ENCODING: " + charset); + } + } + + /** + * Deserialize the Thrift object from a Java string, using the default JVM + * charset encoding. + * + * @param base The object to read into + * @param data The string to read from + */ + public void toString(TBase base, String data) throws TException { + deserialize(base, data.getBytes()); + } +} + diff --git a/lib/java/src/org/apache/thrift/TException.java b/lib/java/src/org/apache/thrift/TException.java new file mode 100644 index 00000000..f84f4812 --- /dev/null +++ b/lib/java/src/org/apache/thrift/TException.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +/** + * Generic exception class for Thrift. + * + */ +public class TException extends Exception { + + private static final long serialVersionUID = 1L; + + public TException() { + super(); + } + + public TException(String message) { + super(message); + } + + public TException(Throwable cause) { + super(cause); + } + + public TException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/lib/java/src/org/apache/thrift/TFieldRequirementType.java b/lib/java/src/org/apache/thrift/TFieldRequirementType.java new file mode 100644 index 00000000..74bac4ef --- /dev/null +++ b/lib/java/src/org/apache/thrift/TFieldRequirementType.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +/** + * Requirement type constants. + * + */ +public final class TFieldRequirementType { + public static final byte REQUIRED = 1; + public static final byte OPTIONAL = 2; + public static final byte DEFAULT = 3; +} diff --git a/lib/java/src/org/apache/thrift/TProcessor.java b/lib/java/src/org/apache/thrift/TProcessor.java new file mode 100644 index 00000000..d79522c3 --- /dev/null +++ b/lib/java/src/org/apache/thrift/TProcessor.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import org.apache.thrift.protocol.TProtocol; + +/** + * A processor is a generic object which operates upon an input stream and + * writes to some output stream. + * + */ +public interface TProcessor { + public boolean process(TProtocol in, TProtocol out) + throws TException; +} diff --git a/lib/java/src/org/apache/thrift/TProcessorFactory.java b/lib/java/src/org/apache/thrift/TProcessorFactory.java new file mode 100644 index 00000000..bcd8a38f --- /dev/null +++ b/lib/java/src/org/apache/thrift/TProcessorFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import org.apache.thrift.transport.TTransport; + +/** + * The default processor factory just returns a singleton + * instance. + */ +public class TProcessorFactory { + + private final TProcessor processor_; + + public TProcessorFactory(TProcessor processor) { + processor_ = processor; + } + + public TProcessor getProcessor(TTransport trans) { + return processor_; + } +} diff --git a/lib/java/src/org/apache/thrift/TSerializer.java b/lib/java/src/org/apache/thrift/TSerializer.java new file mode 100644 index 00000000..4e1ce612 --- /dev/null +++ b/lib/java/src/org/apache/thrift/TSerializer.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import java.io.ByteArrayOutputStream; +import java.io.UnsupportedEncodingException; + +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TIOStreamTransport; + +/** + * Generic utility for easily serializing objects into a byte array or Java + * String. + * + */ +public class TSerializer { + + /** + * This is the byte array that data is actually serialized into + */ + private final ByteArrayOutputStream baos_ = new ByteArrayOutputStream(); + + /** + * This transport wraps that byte array + */ + private final TIOStreamTransport transport_ = new TIOStreamTransport(baos_); + + /** + * Internal protocol used for serializing objects. + */ + private TProtocol protocol_; + + /** + * Create a new TSerializer that uses the TBinaryProtocol by default. + */ + public TSerializer() { + this(new TBinaryProtocol.Factory()); + } + + /** + * Create a new TSerializer. It will use the TProtocol specified by the + * factory that is passed in. + * + * @param protocolFactory Factory to create a protocol + */ + public TSerializer(TProtocolFactory protocolFactory) { + protocol_ = protocolFactory.getProtocol(transport_); + } + + /** + * Serialize the Thrift object into a byte array. The process is simple, + * just clear the byte array output, write the object into it, and grab the + * raw bytes. + * + * @param base The object to serialize + * @return Serialized object in byte[] format + */ + public byte[] serialize(TBase base) throws TException { + baos_.reset(); + base.write(protocol_); + return baos_.toByteArray(); + } + + /** + * Serialize the Thrift object into a Java string, using a specified + * character set for encoding. + * + * @param base The object to serialize + * @param charset Valid JVM charset + * @return Serialized object as a String + */ + public String toString(TBase base, String charset) throws TException { + try { + return new String(serialize(base), charset); + } catch (UnsupportedEncodingException uex) { + throw new TException("JVM DOES NOT SUPPORT ENCODING: " + charset); + } + } + + /** + * Serialize the Thrift object into a Java string, using the default JVM + * charset encoding. + * + * @param base The object to serialize + * @return Serialized object as a String + */ + public String toString(TBase base) throws TException { + return new String(serialize(base)); + } +} + diff --git a/lib/java/src/org/apache/thrift/meta_data/FieldMetaData.java b/lib/java/src/org/apache/thrift/meta_data/FieldMetaData.java new file mode 100644 index 00000000..3e90a8b9 --- /dev/null +++ b/lib/java/src/org/apache/thrift/meta_data/FieldMetaData.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +import java.util.HashMap; +import java.util.Map; +import org.apache.thrift.TBase; + +/** + * This class is used to store meta data about thrift fields. Every field in a + * a struct should have a corresponding instance of this class describing it. + * + */ +public class FieldMetaData implements java.io.Serializable { + public final String fieldName; + public final byte requirementType; + public final FieldValueMetaData valueMetaData; + private static Map, Map> structMap; + + static { + structMap = new HashMap, Map>(); + } + + public FieldMetaData(String name, byte req, FieldValueMetaData vMetaData){ + this.fieldName = name; + this.requirementType = req; + this.valueMetaData = vMetaData; + } + + public static void addStructMetaDataMap(Class sClass, Map map){ + structMap.put(sClass, map); + } + + /** + * Returns a map with metadata (i.e. instances of FieldMetaData) that + * describe the fields of the given class. + * + * @param sClass The TBase class for which the metadata map is requested + */ + public static Map getStructMetaDataMap(Class sClass){ + if (!structMap.containsKey(sClass)){ // Load class if it hasn't been loaded + try{ + sClass.newInstance(); + } catch (InstantiationException e){ + throw new RuntimeException("InstantiationException for TBase class: " + sClass.getName() + ", message: " + e.getMessage()); + } catch (IllegalAccessException e){ + throw new RuntimeException("IllegalAccessException for TBase class: " + sClass.getName() + ", message: " + e.getMessage()); + } + } + return structMap.get(sClass); + } +} diff --git a/lib/java/src/org/apache/thrift/meta_data/FieldValueMetaData.java b/lib/java/src/org/apache/thrift/meta_data/FieldValueMetaData.java new file mode 100644 index 00000000..f72da0cd --- /dev/null +++ b/lib/java/src/org/apache/thrift/meta_data/FieldValueMetaData.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +import org.apache.thrift.protocol.TType; + +/** + * FieldValueMetaData and collection of subclasses to store metadata about + * the value(s) of a field + */ +public class FieldValueMetaData implements java.io.Serializable { + public final byte type; + + public FieldValueMetaData(byte type){ + this.type = type; + } + + public boolean isStruct() { + return type == TType.STRUCT; + } + + public boolean isContainer() { + return type == TType.LIST || type == TType.MAP || type == TType.SET; + } +} diff --git a/lib/java/src/org/apache/thrift/meta_data/ListMetaData.java b/lib/java/src/org/apache/thrift/meta_data/ListMetaData.java new file mode 100644 index 00000000..8e7073bf --- /dev/null +++ b/lib/java/src/org/apache/thrift/meta_data/ListMetaData.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +public class ListMetaData extends FieldValueMetaData { + public final FieldValueMetaData elemMetaData; + + public ListMetaData(byte type, FieldValueMetaData eMetaData){ + super(type); + this.elemMetaData = eMetaData; + } +} diff --git a/lib/java/src/org/apache/thrift/meta_data/MapMetaData.java b/lib/java/src/org/apache/thrift/meta_data/MapMetaData.java new file mode 100644 index 00000000..e7c408c7 --- /dev/null +++ b/lib/java/src/org/apache/thrift/meta_data/MapMetaData.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +public class MapMetaData extends FieldValueMetaData { + public final FieldValueMetaData keyMetaData; + public final FieldValueMetaData valueMetaData; + + public MapMetaData(byte type, FieldValueMetaData kMetaData, FieldValueMetaData vMetaData){ + super(type); + this.keyMetaData = kMetaData; + this.valueMetaData = vMetaData; + } +} diff --git a/lib/java/src/org/apache/thrift/meta_data/SetMetaData.java b/lib/java/src/org/apache/thrift/meta_data/SetMetaData.java new file mode 100644 index 00000000..cf4b96aa --- /dev/null +++ b/lib/java/src/org/apache/thrift/meta_data/SetMetaData.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +public class SetMetaData extends FieldValueMetaData { + public final FieldValueMetaData elemMetaData; + + public SetMetaData(byte type, FieldValueMetaData eMetaData){ + super(type); + this.elemMetaData = eMetaData; + } +} diff --git a/lib/java/src/org/apache/thrift/meta_data/StructMetaData.java b/lib/java/src/org/apache/thrift/meta_data/StructMetaData.java new file mode 100644 index 00000000..b37d21da --- /dev/null +++ b/lib/java/src/org/apache/thrift/meta_data/StructMetaData.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +import org.apache.thrift.TBase; + +public class StructMetaData extends FieldValueMetaData { + public final Class structClass; + + public StructMetaData(byte type, Class sClass){ + super(type); + this.structClass = sClass; + } +} diff --git a/lib/java/src/org/apache/thrift/protocol/TBase64Utils.java b/lib/java/src/org/apache/thrift/protocol/TBase64Utils.java new file mode 100644 index 00000000..37a9fd9f --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TBase64Utils.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** + * Class for encoding and decoding Base64 data. + * + * This class is kept at package level because the interface does no input + * validation and is therefore too low-level for generalized reuse. + * + * Note also that the encoding does not pad with equal signs , as discussed in + * section 2.2 of the RFC (http://www.faqs.org/rfcs/rfc3548.html). Furthermore, + * bad data encountered when decoding is neither rejected or ignored but simply + * results in bad decoded data -- this is not in compliance with the RFC but is + * done in the interest of performance. + * + */ +class TBase64Utils { + + private static final String ENCODE_TABLE = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + /** + * Encode len bytes of data in src at offset srcOff, storing the result into + * dst at offset dstOff. len must be 1, 2, or 3. dst must have at least len+1 + * bytes of space at dstOff. src and dst should not be the same object. This + * method does no validation of the input values in the interest of + * performance. + * + * @param src the source of bytes to encode + * @param srcOff the offset into the source to read the unencoded bytes + * @param len the number of bytes to encode (must be 1, 2, or 3). + * @param dst the destination for the encoding + * @param dstOff the offset into the destination to place the encoded bytes + */ + static final void encode(byte[] src, int srcOff, int len, byte[] dst, + int dstOff) { + dst[dstOff] = (byte)ENCODE_TABLE.charAt((src[srcOff] >> 2) & 0x3F); + if (len == 3) { + dst[dstOff + 1] = + (byte)ENCODE_TABLE.charAt( + ((src[srcOff] << 4) + (src[srcOff+1] >> 4)) & 0x3F); + dst[dstOff + 2] = + (byte)ENCODE_TABLE.charAt( + ((src[srcOff+1] << 2) + (src[srcOff+2] >> 6)) & 0x3F); + dst[dstOff + 3] = + (byte)ENCODE_TABLE.charAt(src[srcOff+2] & 0x3F); + } + else if (len == 2) { + dst[dstOff+1] = + (byte)ENCODE_TABLE.charAt( + ((src[srcOff] << 4) + (src[srcOff+1] >> 4)) & 0x3F); + dst[dstOff + 2] = + (byte)ENCODE_TABLE.charAt((src[srcOff+1] << 2) & 0x3F); + + } + else { // len == 1) { + dst[dstOff + 1] = + (byte)ENCODE_TABLE.charAt((src[srcOff] << 4) & 0x3F); + } + } + + private static final byte[] DECODE_TABLE = { + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,62,-1,-1,-1,63, + 52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-1,-1,-1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, + 15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1, + -1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, + 41,42,43,44,45,46,47,48,49,50,51,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + }; + + /** + * Decode len bytes of data in src at offset srcOff, storing the result into + * dst at offset dstOff. len must be 2, 3, or 4. dst must have at least len-1 + * bytes of space at dstOff. src and dst may be the same object as long as + * dstoff <= srcOff. This method does no validation of the input values in + * the interest of performance. + * + * @param src the source of bytes to decode + * @param srcOff the offset into the source to read the encoded bytes + * @param len the number of bytes to decode (must be 2, 3, or 4) + * @param dst the destination for the decoding + * @param dstOff the offset into the destination to place the decoded bytes + */ + static final void decode(byte[] src, int srcOff, int len, byte[] dst, + int dstOff) { + dst[dstOff] = (byte) + ((DECODE_TABLE[src[srcOff] & 0x0FF] << 2) | + (DECODE_TABLE[src[srcOff+1] & 0x0FF] >> 4)); + if (len > 2) { + dst[dstOff+1] = (byte) + (((DECODE_TABLE[src[srcOff+1] & 0x0FF] << 4) & 0xF0) | + (DECODE_TABLE[src[srcOff+2] & 0x0FF] >> 2)); + if (len > 3) { + dst[dstOff+2] = (byte) + (((DECODE_TABLE[src[srcOff+2] & 0x0FF] << 6) & 0xC0) | + DECODE_TABLE[src[srcOff+3] & 0x0FF]); + } + } + } +} diff --git a/lib/java/src/org/apache/thrift/protocol/TBinaryProtocol.java b/lib/java/src/org/apache/thrift/protocol/TBinaryProtocol.java new file mode 100644 index 00000000..e9bd8b79 --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TBinaryProtocol.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import java.io.UnsupportedEncodingException; + +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransport; + +/** + * Binary protocol implementation for thrift. + * + */ +public class TBinaryProtocol extends TProtocol { + private static final TStruct ANONYMOUS_STRUCT = new TStruct(); + + protected static final int VERSION_MASK = 0xffff0000; + protected static final int VERSION_1 = 0x80010000; + + protected boolean strictRead_ = false; + protected boolean strictWrite_ = true; + + protected int readLength_; + protected boolean checkReadLength_ = false; + + /** + * Factory + */ + public static class Factory implements TProtocolFactory { + protected boolean strictRead_ = false; + protected boolean strictWrite_ = true; + + public Factory() { + this(false, true); + } + + public Factory(boolean strictRead, boolean strictWrite) { + strictRead_ = strictRead; + strictWrite_ = strictWrite; + } + + public TProtocol getProtocol(TTransport trans) { + return new TBinaryProtocol(trans, strictRead_, strictWrite_); + } + } + + /** + * Constructor + */ + public TBinaryProtocol(TTransport trans) { + this(trans, false, true); + } + + public TBinaryProtocol(TTransport trans, boolean strictRead, boolean strictWrite) { + super(trans); + strictRead_ = strictRead; + strictWrite_ = strictWrite; + } + + public void writeMessageBegin(TMessage message) throws TException { + if (strictWrite_) { + int version = VERSION_1 | message.type; + writeI32(version); + writeString(message.name); + writeI32(message.seqid); + } else { + writeString(message.name); + writeByte(message.type); + writeI32(message.seqid); + } + } + + public void writeMessageEnd() {} + + public void writeStructBegin(TStruct struct) {} + + public void writeStructEnd() {} + + public void writeFieldBegin(TField field) throws TException { + writeByte(field.type); + writeI16(field.id); + } + + public void writeFieldEnd() {} + + public void writeFieldStop() throws TException { + writeByte(TType.STOP); + } + + public void writeMapBegin(TMap map) throws TException { + writeByte(map.keyType); + writeByte(map.valueType); + writeI32(map.size); + } + + public void writeMapEnd() {} + + public void writeListBegin(TList list) throws TException { + writeByte(list.elemType); + writeI32(list.size); + } + + public void writeListEnd() {} + + public void writeSetBegin(TSet set) throws TException { + writeByte(set.elemType); + writeI32(set.size); + } + + public void writeSetEnd() {} + + public void writeBool(boolean b) throws TException { + writeByte(b ? (byte)1 : (byte)0); + } + + private byte [] bout = new byte[1]; + public void writeByte(byte b) throws TException { + bout[0] = b; + trans_.write(bout, 0, 1); + } + + private byte[] i16out = new byte[2]; + public void writeI16(short i16) throws TException { + i16out[0] = (byte)(0xff & (i16 >> 8)); + i16out[1] = (byte)(0xff & (i16)); + trans_.write(i16out, 0, 2); + } + + private byte[] i32out = new byte[4]; + public void writeI32(int i32) throws TException { + i32out[0] = (byte)(0xff & (i32 >> 24)); + i32out[1] = (byte)(0xff & (i32 >> 16)); + i32out[2] = (byte)(0xff & (i32 >> 8)); + i32out[3] = (byte)(0xff & (i32)); + trans_.write(i32out, 0, 4); + } + + private byte[] i64out = new byte[8]; + public void writeI64(long i64) throws TException { + i64out[0] = (byte)(0xff & (i64 >> 56)); + i64out[1] = (byte)(0xff & (i64 >> 48)); + i64out[2] = (byte)(0xff & (i64 >> 40)); + i64out[3] = (byte)(0xff & (i64 >> 32)); + i64out[4] = (byte)(0xff & (i64 >> 24)); + i64out[5] = (byte)(0xff & (i64 >> 16)); + i64out[6] = (byte)(0xff & (i64 >> 8)); + i64out[7] = (byte)(0xff & (i64)); + trans_.write(i64out, 0, 8); + } + + public void writeDouble(double dub) throws TException { + writeI64(Double.doubleToLongBits(dub)); + } + + public void writeString(String str) throws TException { + try { + byte[] dat = str.getBytes("UTF-8"); + writeI32(dat.length); + trans_.write(dat, 0, dat.length); + } catch (UnsupportedEncodingException uex) { + throw new TException("JVM DOES NOT SUPPORT UTF-8"); + } + } + + public void writeBinary(byte[] bin) throws TException { + writeI32(bin.length); + trans_.write(bin, 0, bin.length); + } + + /** + * Reading methods. + */ + + public TMessage readMessageBegin() throws TException { + int size = readI32(); + if (size < 0) { + int version = size & VERSION_MASK; + if (version != VERSION_1) { + throw new TProtocolException(TProtocolException.BAD_VERSION, "Bad version in readMessageBegin"); + } + return new TMessage(readString(), (byte)(size & 0x000000ff), readI32()); + } else { + if (strictRead_) { + throw new TProtocolException(TProtocolException.BAD_VERSION, "Missing version in readMessageBegin, old client?"); + } + return new TMessage(readStringBody(size), readByte(), readI32()); + } + } + + public void readMessageEnd() {} + + public TStruct readStructBegin() { + return ANONYMOUS_STRUCT; + } + + public void readStructEnd() {} + + public TField readFieldBegin() throws TException { + byte type = readByte(); + short id = type == TType.STOP ? 0 : readI16(); + return new TField("", type, id); + } + + public void readFieldEnd() {} + + public TMap readMapBegin() throws TException { + return new TMap(readByte(), readByte(), readI32()); + } + + public void readMapEnd() {} + + public TList readListBegin() throws TException { + return new TList(readByte(), readI32()); + } + + public void readListEnd() {} + + public TSet readSetBegin() throws TException { + return new TSet(readByte(), readI32()); + } + + public void readSetEnd() {} + + public boolean readBool() throws TException { + return (readByte() == 1); + } + + private byte[] bin = new byte[1]; + public byte readByte() throws TException { + readAll(bin, 0, 1); + return bin[0]; + } + + private byte[] i16rd = new byte[2]; + public short readI16() throws TException { + readAll(i16rd, 0, 2); + return + (short) + (((i16rd[0] & 0xff) << 8) | + ((i16rd[1] & 0xff))); + } + + private byte[] i32rd = new byte[4]; + public int readI32() throws TException { + readAll(i32rd, 0, 4); + return + ((i32rd[0] & 0xff) << 24) | + ((i32rd[1] & 0xff) << 16) | + ((i32rd[2] & 0xff) << 8) | + ((i32rd[3] & 0xff)); + } + + private byte[] i64rd = new byte[8]; + public long readI64() throws TException { + readAll(i64rd, 0, 8); + return + ((long)(i64rd[0] & 0xff) << 56) | + ((long)(i64rd[1] & 0xff) << 48) | + ((long)(i64rd[2] & 0xff) << 40) | + ((long)(i64rd[3] & 0xff) << 32) | + ((long)(i64rd[4] & 0xff) << 24) | + ((long)(i64rd[5] & 0xff) << 16) | + ((long)(i64rd[6] & 0xff) << 8) | + ((long)(i64rd[7] & 0xff)); + } + + public double readDouble() throws TException { + return Double.longBitsToDouble(readI64()); + } + + public String readString() throws TException { + int size = readI32(); + return readStringBody(size); + } + + public String readStringBody(int size) throws TException { + try { + checkReadLength(size); + byte[] buf = new byte[size]; + trans_.readAll(buf, 0, size); + return new String(buf, "UTF-8"); + } catch (UnsupportedEncodingException uex) { + throw new TException("JVM DOES NOT SUPPORT UTF-8"); + } + } + + public byte[] readBinary() throws TException { + int size = readI32(); + checkReadLength(size); + byte[] buf = new byte[size]; + trans_.readAll(buf, 0, size); + return buf; + } + + private int readAll(byte[] buf, int off, int len) throws TException { + checkReadLength(len); + return trans_.readAll(buf, off, len); + } + + public void setReadLength(int readLength) { + readLength_ = readLength; + checkReadLength_ = true; + } + + protected void checkReadLength(int length) throws TException { + if (checkReadLength_) { + readLength_ -= length; + if (readLength_ < 0) { + throw new TException("Message length exceeded: " + length); + } + } + } + +} diff --git a/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java b/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java new file mode 100755 index 00000000..e2d0bfdc --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TCompactProtocol.java @@ -0,0 +1,741 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.thrift.protocol; + +import java.util.Stack; +import java.io.UnsupportedEncodingException; + +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.TException; + +/** + * TCompactProtocol2 is the Java implementation of the compact protocol specified + * in THRIFT-110. The fundamental approach to reducing the overhead of + * structures is a) use variable-length integers all over the place and b) make + * use of unused bits wherever possible. Your savings will obviously vary + * based on the specific makeup of your structs, but in general, the more + * fields, nested structures, short strings and collections, and low-value i32 + * and i64 fields you have, the more benefit you'll see. + */ +public final class TCompactProtocol extends TProtocol { + + private final static TStruct ANONYMOUS_STRUCT = new TStruct(""); + private final static TField TSTOP = new TField("", TType.STOP, (short)0); + + private final static byte[] ttypeToCompactType = new byte[16]; + + static { + ttypeToCompactType[TType.STOP] = TType.STOP; + ttypeToCompactType[TType.BOOL] = Types.BOOLEAN_TRUE; + ttypeToCompactType[TType.BYTE] = Types.BYTE; + ttypeToCompactType[TType.I16] = Types.I16; + ttypeToCompactType[TType.I32] = Types.I32; + ttypeToCompactType[TType.I64] = Types.I64; + ttypeToCompactType[TType.DOUBLE] = Types.DOUBLE; + ttypeToCompactType[TType.STRING] = Types.BINARY; + ttypeToCompactType[TType.LIST] = Types.LIST; + ttypeToCompactType[TType.SET] = Types.SET; + ttypeToCompactType[TType.MAP] = Types.MAP; + ttypeToCompactType[TType.STRUCT] = Types.STRUCT; + } + + /** + * TProtocolFactory that produces TCompactProtocols. + */ + public static class Factory implements TProtocolFactory { + public Factory() {} + + public TProtocol getProtocol(TTransport trans) { + return new TCompactProtocol(trans); + } + } + + private static final byte PROTOCOL_ID = (byte)0x82; + private static final byte VERSION = 1; + private static final byte VERSION_MASK = 0x1f; // 0001 1111 + private static final byte TYPE_MASK = (byte)0xE0; // 1110 0000 + private static final int TYPE_SHIFT_AMOUNT = 5; + + /** + * All of the on-wire type codes. + */ + private static class Types { + public static final byte BOOLEAN_TRUE = 0x01; + public static final byte BOOLEAN_FALSE = 0x02; + public static final byte BYTE = 0x03; + public static final byte I16 = 0x04; + public static final byte I32 = 0x05; + public static final byte I64 = 0x06; + public static final byte DOUBLE = 0x07; + public static final byte BINARY = 0x08; + public static final byte LIST = 0x09; + public static final byte SET = 0x0A; + public static final byte MAP = 0x0B; + public static final byte STRUCT = 0x0C; + } + + /** + * Used to keep track of the last field for the current and previous structs, + * so we can do the delta stuff. + */ + private Stack lastField_ = new Stack(); + + private short lastFieldId_ = 0; + + /** + * If we encounter a boolean field begin, save the TField here so it can + * have the value incorporated. + */ + private TField booleanField_ = null; + + /** + * If we read a field header, and it's a boolean field, save the boolean + * value here so that readBool can use it. + */ + private Boolean boolValue_ = null; + + /** + * Create a TCompactProtocol. + * + * @param transport the TTransport object to read from or write to. + */ + public TCompactProtocol(TTransport transport) { + super(transport); + } + + + // + // Public Writing methods. + // + + /** + * Write a message header to the wire. Compact Protocol messages contain the + * protocol version so we can migrate forwards in the future if need be. + */ + public void writeMessageBegin(TMessage message) throws TException { + writeByteDirect(PROTOCOL_ID); + writeByteDirect((VERSION & VERSION_MASK) | ((message.type << TYPE_SHIFT_AMOUNT) & TYPE_MASK)); + writeVarint32(message.seqid); + writeString(message.name); + } + + /** + * Write a struct begin. This doesn't actually put anything on the wire. We + * use it as an opportunity to put special placeholder markers on the field + * stack so we can get the field id deltas correct. + */ + public void writeStructBegin(TStruct struct) throws TException { + lastField_.push(lastFieldId_); + lastFieldId_ = 0; + } + + /** + * Write a struct end. This doesn't actually put anything on the wire. We use + * this as an opportunity to pop the last field from the current struct off + * of the field stack. + */ + public void writeStructEnd() throws TException { + lastFieldId_ = lastField_.pop(); + } + + /** + * Write a field header containing the field id and field type. If the + * difference between the current field id and the last one is small (< 15), + * then the field id will be encoded in the 4 MSB as a delta. Otherwise, the + * field id will follow the type header as a zigzag varint. + */ + public void writeFieldBegin(TField field) throws TException { + if (field.type == TType.BOOL) { + // we want to possibly include the value, so we'll wait. + booleanField_ = field; + } else { + writeFieldBeginInternal(field, (byte)-1); + } + } + + /** + * The workhorse of writeFieldBegin. It has the option of doing a + * 'type override' of the type header. This is used specifically in the + * boolean field case. + */ + private void writeFieldBeginInternal(TField field, byte typeOverride) throws TException { + // short lastField = lastField_.pop(); + + // if there's a type override, use that. + byte typeToWrite = typeOverride == -1 ? getCompactType(field.type) : typeOverride; + + // check if we can use delta encoding for the field id + if (field.id > lastFieldId_ && field.id - lastFieldId_ <= 15) { + // write them together + writeByteDirect((field.id - lastFieldId_) << 4 | typeToWrite); + } else { + // write them separate + writeByteDirect(typeToWrite); + writeI16(field.id); + } + + lastFieldId_ = field.id; + // lastField_.push(field.id); + } + + /** + * Write the STOP symbol so we know there are no more fields in this struct. + */ + public void writeFieldStop() throws TException { + writeByteDirect(TType.STOP); + } + + /** + * Write a map header. If the map is empty, omit the key and value type + * headers, as we don't need any additional information to skip it. + */ + public void writeMapBegin(TMap map) throws TException { + if (map.size == 0) { + writeByteDirect(0); + } else { + writeVarint32(map.size); + writeByteDirect(getCompactType(map.keyType) << 4 | getCompactType(map.valueType)); + } + } + + /** + * Write a list header. + */ + public void writeListBegin(TList list) throws TException { + writeCollectionBegin(list.elemType, list.size); + } + + /** + * Write a set header. + */ + public void writeSetBegin(TSet set) throws TException { + writeCollectionBegin(set.elemType, set.size); + } + + /** + * Write a boolean value. Potentially, this could be a boolean field, in + * which case the field header info isn't written yet. If so, decide what the + * right type header is for the value and then write the field header. + * Otherwise, write a single byte. + */ + public void writeBool(boolean b) throws TException { + if (booleanField_ != null) { + // we haven't written the field header yet + writeFieldBeginInternal(booleanField_, b ? Types.BOOLEAN_TRUE : Types.BOOLEAN_FALSE); + booleanField_ = null; + } else { + // we're not part of a field, so just write the value. + writeByteDirect(b ? Types.BOOLEAN_TRUE : Types.BOOLEAN_FALSE); + } + } + + /** + * Write a byte. Nothing to see here! + */ + public void writeByte(byte b) throws TException { + writeByteDirect(b); + } + + /** + * Write an I16 as a zigzag varint. + */ + public void writeI16(short i16) throws TException { + writeVarint32(intToZigZag(i16)); + } + + /** + * Write an i32 as a zigzag varint. + */ + public void writeI32(int i32) throws TException { + writeVarint32(intToZigZag(i32)); + } + + /** + * Write an i64 as a zigzag varint. + */ + public void writeI64(long i64) throws TException { + writeVarint64(longToZigzag(i64)); + } + + /** + * Write a double to the wire as 8 bytes. + */ + public void writeDouble(double dub) throws TException { + byte[] data = new byte[]{0, 0, 0, 0, 0, 0, 0, 0}; + fixedLongToBytes(Double.doubleToLongBits(dub), data, 0); + trans_.write(data); + } + + /** + * Write a string to the wire with a varint size preceeding. + */ + public void writeString(String str) throws TException { + try { + writeBinary(str.getBytes("UTF-8")); + } catch (UnsupportedEncodingException e) { + throw new TException("UTF-8 not supported!"); + } + } + + /** + * Write a byte array, using a varint for the size. + */ + public void writeBinary(byte[] bin) throws TException { + writeVarint32(bin.length); + trans_.write(bin); + } + + // + // These methods are called by structs, but don't actually have any wire + // output or purpose. + // + + public void writeMessageEnd() throws TException {} + public void writeMapEnd() throws TException {} + public void writeListEnd() throws TException {} + public void writeSetEnd() throws TException {} + public void writeFieldEnd() throws TException {} + + // + // Internal writing methods + // + + /** + * Abstract method for writing the start of lists and sets. List and sets on + * the wire differ only by the type indicator. + */ + protected void writeCollectionBegin(byte elemType, int size) throws TException { + if (size <= 14) { + writeByteDirect(size << 4 | getCompactType(elemType)); + } else { + writeByteDirect(0xf0 | getCompactType(elemType)); + writeVarint32(size); + } + } + + /** + * Write an i32 as a varint. Results in 1-5 bytes on the wire. + * TODO: make a permanent buffer like writeVarint64? + */ + byte[] i32buf = new byte[5]; + private void writeVarint32(int n) throws TException { + int idx = 0; + while (true) { + if ((n & ~0x7F) == 0) { + i32buf[idx++] = (byte)n; + // writeByteDirect((byte)n); + break; + // return; + } else { + i32buf[idx++] = (byte)((n & 0x7F) | 0x80); + // writeByteDirect((byte)((n & 0x7F) | 0x80)); + n >>>= 7; + } + } + trans_.write(i32buf, 0, idx); + } + + /** + * Write an i64 as a varint. Results in 1-10 bytes on the wire. + */ + byte[] varint64out = new byte[10]; + private void writeVarint64(long n) throws TException { + int idx = 0; + while (true) { + if ((n & ~0x7FL) == 0) { + varint64out[idx++] = (byte)n; + break; + } else { + varint64out[idx++] = ((byte)((n & 0x7F) | 0x80)); + n >>>= 7; + } + } + trans_.write(varint64out, 0, idx); + } + + /** + * Convert l into a zigzag long. This allows negative numbers to be + * represented compactly as a varint. + */ + private long longToZigzag(long l) { + return (l << 1) ^ (l >> 63); + } + + /** + * Convert n into a zigzag int. This allows negative numbers to be + * represented compactly as a varint. + */ + private int intToZigZag(int n) { + return (n << 1) ^ (n >> 31); + } + + /** + * Convert a long into little-endian bytes in buf starting at off and going + * until off+7. + */ + private void fixedLongToBytes(long n, byte[] buf, int off) { + buf[off+0] = (byte)( n & 0xff); + buf[off+1] = (byte)((n >> 8 ) & 0xff); + buf[off+2] = (byte)((n >> 16) & 0xff); + buf[off+3] = (byte)((n >> 24) & 0xff); + buf[off+4] = (byte)((n >> 32) & 0xff); + buf[off+5] = (byte)((n >> 40) & 0xff); + buf[off+6] = (byte)((n >> 48) & 0xff); + buf[off+7] = (byte)((n >> 56) & 0xff); + } + + /** + * Writes a byte without any possiblity of all that field header nonsense. + * Used internally by other writing methods that know they need to write a byte. + */ + private byte[] byteDirectBuffer = new byte[1]; + private void writeByteDirect(byte b) throws TException { + byteDirectBuffer[0] = b; + trans_.write(byteDirectBuffer); + } + + /** + * Writes a byte without any possiblity of all that field header nonsense. + */ + private void writeByteDirect(int n) throws TException { + writeByteDirect((byte)n); + } + + + // + // Reading methods. + // + + /** + * Read a message header. + */ + public TMessage readMessageBegin() throws TException { + byte protocolId = readByte(); + if (protocolId != PROTOCOL_ID) { + throw new TProtocolException("Expected protocol id " + Integer.toHexString(PROTOCOL_ID) + " but got " + Integer.toHexString(protocolId)); + } + byte versionAndType = readByte(); + byte version = (byte)(versionAndType & VERSION_MASK); + if (version != VERSION) { + throw new TProtocolException("Expected version " + VERSION + " but got " + version); + } + byte type = (byte)((versionAndType >> TYPE_SHIFT_AMOUNT) & 0x03); + int seqid = readVarint32(); + String messageName = readString(); + return new TMessage(messageName, type, seqid); + } + + /** + * Read a struct begin. There's nothing on the wire for this, but it is our + * opportunity to push a new struct begin marker onto the field stack. + */ + public TStruct readStructBegin() throws TException { + lastField_.push(lastFieldId_); + lastFieldId_ = 0; + return ANONYMOUS_STRUCT; + } + + /** + * Doesn't actually consume any wire data, just removes the last field for + * this struct from the field stack. + */ + public void readStructEnd() throws TException { + // consume the last field we read off the wire. + lastFieldId_ = lastField_.pop(); + } + + /** + * Read a field header off the wire. + */ + public TField readFieldBegin() throws TException { + byte type = readByte(); + + // if it's a stop, then we can return immediately, as the struct is over. + if ((type & 0x0f) == TType.STOP) { + return TSTOP; + } + + short fieldId; + + // mask off the 4 MSB of the type header. it could contain a field id delta. + short modifier = (short)((type & 0xf0) >> 4); + if (modifier == 0) { + // not a delta. look ahead for the zigzag varint field id. + fieldId = readI16(); + } else { + // has a delta. add the delta to the last read field id. + fieldId = (short)(lastFieldId_ + modifier); + } + + TField field = new TField("", getTType((byte)(type & 0x0f)), fieldId); + + // if this happens to be a boolean field, the value is encoded in the type + if (isBoolType(type)) { + // save the boolean value in a special instance variable. + boolValue_ = (byte)(type & 0x0f) == Types.BOOLEAN_TRUE ? Boolean.TRUE : Boolean.FALSE; + } + + // push the new field onto the field stack so we can keep the deltas going. + lastFieldId_ = field.id; + return field; + } + + /** + * Read a map header off the wire. If the size is zero, skip reading the key + * and value type. This means that 0-length maps will yield TMaps without the + * "correct" types. + */ + public TMap readMapBegin() throws TException { + int size = readVarint32(); + byte keyAndValueType = size == 0 ? 0 : readByte(); + return new TMap(getTType((byte)(keyAndValueType >> 4)), getTType((byte)(keyAndValueType & 0xf)), size); + } + + /** + * Read a list header off the wire. If the list size is 0-14, the size will + * be packed into the element type header. If it's a longer list, the 4 MSB + * of the element type header will be 0xF, and a varint will follow with the + * true size. + */ + public TList readListBegin() throws TException { + byte size_and_type = readByte(); + int size = (size_and_type >> 4) & 0x0f; + if (size == 15) { + size = readVarint32(); + } + byte type = getTType(size_and_type); + return new TList(type, size); + } + + /** + * Read a set header off the wire. If the set size is 0-14, the size will + * be packed into the element type header. If it's a longer set, the 4 MSB + * of the element type header will be 0xF, and a varint will follow with the + * true size. + */ + public TSet readSetBegin() throws TException { + return new TSet(readListBegin()); + } + + /** + * Read a boolean off the wire. If this is a boolean field, the value should + * already have been read during readFieldBegin, so we'll just consume the + * pre-stored value. Otherwise, read a byte. + */ + public boolean readBool() throws TException { + if (boolValue_ != null) { + boolean result = boolValue_.booleanValue(); + boolValue_ = null; + return result; + } + return readByte() == Types.BOOLEAN_TRUE; + } + + byte[] byteRawBuf = new byte[1]; + /** + * Read a single byte off the wire. Nothing interesting here. + */ + public byte readByte() throws TException { + trans_.read(byteRawBuf, 0, 1); + return byteRawBuf[0]; + } + + /** + * Read an i16 from the wire as a zigzag varint. + */ + public short readI16() throws TException { + return (short)zigzagToInt(readVarint32()); + } + + /** + * Read an i32 from the wire as a zigzag varint. + */ + public int readI32() throws TException { + return zigzagToInt(readVarint32()); + } + + /** + * Read an i64 from the wire as a zigzag varint. + */ + public long readI64() throws TException { + return zigzagToLong(readVarint64()); + } + + /** + * No magic here - just read a double off the wire. + */ + public double readDouble() throws TException { + byte[] longBits = new byte[8]; + trans_.read(longBits, 0, 8); + return Double.longBitsToDouble(bytesToLong(longBits)); + } + + /** + * Reads a byte[] (via readBinary), and then UTF-8 decodes it. + */ + public String readString() throws TException { + try { + return new String(readBinary(), "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new TException("UTF-8 not supported!"); + } + } + + /** + * Read a byte[] from the wire. + */ + public byte[] readBinary() throws TException { + int length = readVarint32(); + if (length == 0) return new byte[0]; + + byte[] buf = new byte[length]; + trans_.read(buf, 0, length); + return buf; + } + + + // + // These methods are here for the struct to call, but don't have any wire + // encoding. + // + public void readMessageEnd() throws TException {} + public void readFieldEnd() throws TException {} + public void readMapEnd() throws TException {} + public void readListEnd() throws TException {} + public void readSetEnd() throws TException {} + + // + // Internal reading methods + // + + /** + * Read an i32 from the wire as a varint. The MSB of each byte is set + * if there is another byte to follow. This can read up to 5 bytes. + */ + private int readVarint32() throws TException { + // if the wire contains the right stuff, this will just truncate the i64 we + // read and get us the right sign. + return (int)readVarint64(); + } + + /** + * Read an i64 from the wire as a proper varint. The MSB of each byte is set + * if there is another byte to follow. This can read up to 10 bytes. + */ + private long readVarint64() throws TException { + int shift = 0; + long result = 0; + while (true) { + byte b = readByte(); + result |= (long) (b & 0x7f) << shift; + if ((b & 0x80) != 0x80) break; + shift +=7; + } + return result; + } + + // + // encoding helpers + // + + /** + * Convert from zigzag int to int. + */ + private int zigzagToInt(int n) { + return (n >>> 1) ^ -(n & 1); + } + + /** + * Convert from zigzag long to long. + */ + private long zigzagToLong(long n) { + return (n >>> 1) ^ -(n & 1); + } + + /** + * Note that it's important that the mask bytes are long literals, + * otherwise they'll default to ints, and when you shift an int left 56 bits, + * you just get a messed up int. + */ + private long bytesToLong(byte[] bytes) { + return + ((bytes[7] & 0xffL) << 56) | + ((bytes[6] & 0xffL) << 48) | + ((bytes[5] & 0xffL) << 40) | + ((bytes[4] & 0xffL) << 32) | + ((bytes[3] & 0xffL) << 24) | + ((bytes[2] & 0xffL) << 16) | + ((bytes[1] & 0xffL) << 8) | + ((bytes[0] & 0xffL)); + } + + // + // type testing and converting + // + + private boolean isBoolType(byte b) { + return (b & 0x0f) == Types.BOOLEAN_TRUE || (b & 0x0f) == Types.BOOLEAN_FALSE; + } + + /** + * Given a TCompactProtocol.Types constant, convert it to its corresponding + * TType value. + */ + private byte getTType(byte type) { + switch ((byte)(type & 0x0f)) { + case TType.STOP: + return TType.STOP; + case Types.BOOLEAN_FALSE: + case Types.BOOLEAN_TRUE: + return TType.BOOL; + case Types.BYTE: + return TType.BYTE; + case Types.I16: + return TType.I16; + case Types.I32: + return TType.I32; + case Types.I64: + return TType.I64; + case Types.DOUBLE: + return TType.DOUBLE; + case Types.BINARY: + return TType.STRING; + case Types.LIST: + return TType.LIST; + case Types.SET: + return TType.SET; + case Types.MAP: + return TType.MAP; + case Types.STRUCT: + return TType.STRUCT; + default: + throw new RuntimeException("don't know what type: " + (byte)(type & 0x0f)); + } + } + + /** + * Given a TType value, find the appropriate TCompactProtocol.Types constant. + */ + private byte getCompactType(byte ttype) { + return ttypeToCompactType[ttype]; + } + +} diff --git a/lib/java/src/org/apache/thrift/protocol/TField.java b/lib/java/src/org/apache/thrift/protocol/TField.java new file mode 100644 index 00000000..03affdaa --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TField.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** + * Helper class that encapsulates field metadata. + * + */ +public class TField { + public TField() { + this("", TType.STOP, (short)0); + } + + public TField(String n, byte t, short i) { + name = n; + type = t; + id = i; + } + + public final String name; + public final byte type; + public final short id; + + public String toString() { + return ""; + } + + public boolean equals(TField otherField) { + return type == otherField.type && id == otherField.id; + } +} diff --git a/lib/java/src/org/apache/thrift/protocol/TJSONProtocol.java b/lib/java/src/org/apache/thrift/protocol/TJSONProtocol.java new file mode 100644 index 00000000..631c6a5b --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TJSONProtocol.java @@ -0,0 +1,927 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import org.apache.thrift.TException; +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.transport.TTransport; +import java.io.UnsupportedEncodingException; +import java.util.Stack; + +/** + * JSON protocol implementation for thrift. + * + * This is a full-featured protocol supporting write and read. + * + * Please see the C++ class header for a detailed description of the + * protocol's wire format. + * + */ +public class TJSONProtocol extends TProtocol { + + /** + * Factory for JSON protocol objects + */ + public static class Factory implements TProtocolFactory { + + public TProtocol getProtocol(TTransport trans) { + return new TJSONProtocol(trans); + } + + } + + private static final byte[] COMMA = new byte[] {','}; + private static final byte[] COLON = new byte[] {':'}; + private static final byte[] LBRACE = new byte[] {'{'}; + private static final byte[] RBRACE = new byte[] {'}'}; + private static final byte[] LBRACKET = new byte[] {'['}; + private static final byte[] RBRACKET = new byte[] {']'}; + private static final byte[] QUOTE = new byte[] {'"'}; + private static final byte[] BACKSLASH = new byte[] {'\\'}; + private static final byte[] ZERO = new byte[] {'0'}; + + private static final byte[] ESCSEQ = new byte[] {'\\','u','0','0'}; + + private static final long VERSION = 1; + + private static final byte[] JSON_CHAR_TABLE = { + /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ + 0, 0, 0, 0, 0, 0, 0, 0,'b','t','n', 0,'f','r', 0, 0, // 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1 + 1, 1,'"', 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2 + }; + + private static final String ESCAPE_CHARS = "\"\\bfnrt"; + + private static final byte[] ESCAPE_CHAR_VALS = { + '"', '\\', '\b', '\f', '\n', '\r', '\t', + }; + + private static final int DEF_STRING_SIZE = 16; + + private static final byte[] NAME_BOOL = new byte[] {'t', 'f'}; + private static final byte[] NAME_BYTE = new byte[] {'i','8'}; + private static final byte[] NAME_I16 = new byte[] {'i','1','6'}; + private static final byte[] NAME_I32 = new byte[] {'i','3','2'}; + private static final byte[] NAME_I64 = new byte[] {'i','6','4'}; + private static final byte[] NAME_DOUBLE = new byte[] {'d','b','l'}; + private static final byte[] NAME_STRUCT = new byte[] {'r','e','c'}; + private static final byte[] NAME_STRING = new byte[] {'s','t','r'}; + private static final byte[] NAME_MAP = new byte[] {'m','a','p'}; + private static final byte[] NAME_LIST = new byte[] {'l','s','t'}; + private static final byte[] NAME_SET = new byte[] {'s','e','t'}; + + private static final TStruct ANONYMOUS_STRUCT = new TStruct(); + + private static final byte[] getTypeNameForTypeID(byte typeID) + throws TException { + switch (typeID) { + case TType.BOOL: + return NAME_BOOL; + case TType.BYTE: + return NAME_BYTE; + case TType.I16: + return NAME_I16; + case TType.I32: + return NAME_I32; + case TType.I64: + return NAME_I64; + case TType.DOUBLE: + return NAME_DOUBLE; + case TType.STRING: + return NAME_STRING; + case TType.STRUCT: + return NAME_STRUCT; + case TType.MAP: + return NAME_MAP; + case TType.SET: + return NAME_SET; + case TType.LIST: + return NAME_LIST; + default: + throw new TProtocolException(TProtocolException.NOT_IMPLEMENTED, + "Unrecognized type"); + } + } + + private static final byte getTypeIDForTypeName(byte[] name) + throws TException { + byte result = TType.STOP; + if (name.length > 1) { + switch (name[0]) { + case 'd': + result = TType.DOUBLE; + break; + case 'i': + switch (name[1]) { + case '8': + result = TType.BYTE; + break; + case '1': + result = TType.I16; + break; + case '3': + result = TType.I32; + break; + case '6': + result = TType.I64; + break; + } + break; + case 'l': + result = TType.LIST; + break; + case 'm': + result = TType.MAP; + break; + case 'r': + result = TType.STRUCT; + break; + case 's': + if (name[1] == 't') { + result = TType.STRING; + } + else if (name[1] == 'e') { + result = TType.SET; + } + break; + case 't': + result = TType.BOOL; + break; + } + } + if (result == TType.STOP) { + throw new TProtocolException(TProtocolException.NOT_IMPLEMENTED, + "Unrecognized type"); + } + return result; + } + + // Base class for tracking JSON contexts that may require inserting/reading + // additional JSON syntax characters + // This base context does nothing. + protected class JSONBaseContext { + protected void write() throws TException {} + + protected void read() throws TException {} + + protected boolean escapeNum() { return false; } + } + + // Context for JSON lists. Will insert/read commas before each item except + // for the first one + protected class JSONListContext extends JSONBaseContext { + private boolean first_ = true; + + @Override + protected void write() throws TException { + if (first_) { + first_ = false; + } else { + trans_.write(COMMA); + } + } + + @Override + protected void read() throws TException { + if (first_) { + first_ = false; + } else { + readJSONSyntaxChar(COMMA); + } + } + } + + // Context for JSON records. Will insert/read colons before the value portion + // of each record pair, and commas before each key except the first. In + // addition, will indicate that numbers in the key position need to be + // escaped in quotes (since JSON keys must be strings). + protected class JSONPairContext extends JSONBaseContext { + private boolean first_ = true; + private boolean colon_ = true; + + @Override + protected void write() throws TException { + if (first_) { + first_ = false; + colon_ = true; + } else { + trans_.write(colon_ ? COLON : COMMA); + colon_ = !colon_; + } + } + + @Override + protected void read() throws TException { + if (first_) { + first_ = false; + colon_ = true; + } else { + readJSONSyntaxChar(colon_ ? COLON : COMMA); + colon_ = !colon_; + } + } + + @Override + protected boolean escapeNum() { + return colon_; + } + } + + // Holds up to one byte from the transport + protected class LookaheadReader { + + private boolean hasData_; + private byte[] data_ = new byte[1]; + + // Return and consume the next byte to be read, either taking it from the + // data buffer if present or getting it from the transport otherwise. + protected byte read() throws TException { + if (hasData_) { + hasData_ = false; + } + else { + trans_.readAll(data_, 0, 1); + } + return data_[0]; + } + + // Return the next byte to be read without consuming, filling the data + // buffer if it has not been filled already. + protected byte peek() throws TException { + if (!hasData_) { + trans_.readAll(data_, 0, 1); + } + hasData_ = true; + return data_[0]; + } + } + + // Stack of nested contexts that we may be in + private Stack contextStack_ = new Stack(); + + // Current context that we are in + private JSONBaseContext context_ = new JSONBaseContext(); + + // Reader that manages a 1-byte buffer + private LookaheadReader reader_ = new LookaheadReader(); + + // Push a new JSON context onto the stack. + private void pushContext(JSONBaseContext c) { + contextStack_.push(context_); + context_ = c; + } + + // Pop the last JSON context off the stack + private void popContext() { + context_ = contextStack_.pop(); + } + + /** + * Constructor + */ + public TJSONProtocol(TTransport trans) { + super(trans); + } + + // Temporary buffer used by several methods + private byte[] tmpbuf_ = new byte[4]; + + // Read a byte that must match b[0]; otherwise an excpetion is thrown. + // Marked protected to avoid synthetic accessor in JSONListContext.read + // and JSONPairContext.read + protected void readJSONSyntaxChar(byte[] b) throws TException { + byte ch = reader_.read(); + if (ch != b[0]) { + throw new TProtocolException(TProtocolException.INVALID_DATA, + "Unexpected character:" + (char)ch); + } + } + + // Convert a byte containing a hex char ('0'-'9' or 'a'-'f') into its + // corresponding hex value + private static final byte hexVal(byte ch) throws TException { + if ((ch >= '0') && (ch <= '9')) { + return (byte)((char)ch - '0'); + } + else if ((ch >= 'a') && (ch <= 'f')) { + return (byte)((char)ch - 'a'); + } + else { + throw new TProtocolException(TProtocolException.INVALID_DATA, + "Expected hex character"); + } + } + + // Convert a byte containing a hex value to its corresponding hex character + private static final byte hexChar(byte val) { + val &= 0x0F; + if (val < 10) { + return (byte)((char)val + '0'); + } + else { + return (byte)((char)val + 'a'); + } + } + + // Write the bytes in array buf as a JSON characters, escaping as needed + private void writeJSONString(byte[] b) throws TException { + context_.write(); + trans_.write(QUOTE); + int len = b.length; + for (int i = 0; i < len; i++) { + if ((b[i] & 0x00FF) >= 0x30) { + if (b[i] == BACKSLASH[0]) { + trans_.write(BACKSLASH); + trans_.write(BACKSLASH); + } + else { + trans_.write(b, i, 1); + } + } + else { + tmpbuf_[0] = JSON_CHAR_TABLE[b[i]]; + if (tmpbuf_[0] == 1) { + trans_.write(b, i, 1); + } + else if (tmpbuf_[0] > 1) { + trans_.write(BACKSLASH); + trans_.write(tmpbuf_, 0, 1); + } + else { + trans_.write(ESCSEQ); + tmpbuf_[0] = hexChar((byte)(b[i] >> 4)); + tmpbuf_[1] = hexChar(b[i]); + trans_.write(tmpbuf_, 0, 2); + } + } + } + trans_.write(QUOTE); + } + + // Write out number as a JSON value. If the context dictates so, it will be + // wrapped in quotes to output as a JSON string. + private void writeJSONInteger(long num) throws TException { + context_.write(); + String str = Long.toString(num); + boolean escapeNum = context_.escapeNum(); + if (escapeNum) { + trans_.write(QUOTE); + } + try { + byte[] buf = str.getBytes("UTF-8"); + trans_.write(buf); + } catch (UnsupportedEncodingException uex) { + throw new TException("JVM DOES NOT SUPPORT UTF-8"); + } + if (escapeNum) { + trans_.write(QUOTE); + } + } + + // Write out a double as a JSON value. If it is NaN or infinity or if the + // context dictates escaping, write out as JSON string. + private void writeJSONDouble(double num) throws TException { + context_.write(); + String str = Double.toString(num); + boolean special = false; + switch (str.charAt(0)) { + case 'N': // NaN + case 'I': // Infinity + special = true; + break; + case '-': + if (str.charAt(1) == 'I') { // -Infinity + special = true; + } + break; + } + + boolean escapeNum = special || context_.escapeNum(); + if (escapeNum) { + trans_.write(QUOTE); + } + try { + byte[] b = str.getBytes("UTF-8"); + trans_.write(b, 0, b.length); + } catch (UnsupportedEncodingException uex) { + throw new TException("JVM DOES NOT SUPPORT UTF-8"); + } + if (escapeNum) { + trans_.write(QUOTE); + } + } + + // Write out contents of byte array b as a JSON string with base-64 encoded + // data + private void writeJSONBase64(byte[] b) throws TException { + context_.write(); + trans_.write(QUOTE); + int len = b.length; + int off = 0; + while (len >= 3) { + // Encode 3 bytes at a time + TBase64Utils.encode(b, off, 3, tmpbuf_, 0); + trans_.write(tmpbuf_, 0, 4); + off += 3; + len -= 3; + } + if (len > 0) { + // Encode remainder + TBase64Utils.encode(b, off, len, tmpbuf_, 0); + trans_.write(tmpbuf_, 0, len + 1); + } + trans_.write(QUOTE); + } + + private void writeJSONObjectStart() throws TException { + context_.write(); + trans_.write(LBRACE); + pushContext(new JSONPairContext()); + } + + private void writeJSONObjectEnd() throws TException { + popContext(); + trans_.write(RBRACE); + } + + private void writeJSONArrayStart() throws TException { + context_.write(); + trans_.write(LBRACKET); + pushContext(new JSONListContext()); + } + + private void writeJSONArrayEnd() throws TException { + popContext(); + trans_.write(RBRACKET); + } + + @Override + public void writeMessageBegin(TMessage message) throws TException { + writeJSONArrayStart(); + writeJSONInteger(VERSION); + try { + byte[] b = message.name.getBytes("UTF-8"); + writeJSONString(b); + } catch (UnsupportedEncodingException uex) { + throw new TException("JVM DOES NOT SUPPORT UTF-8"); + } + writeJSONInteger(message.type); + writeJSONInteger(message.seqid); + } + + @Override + public void writeMessageEnd() throws TException { + writeJSONArrayEnd(); + } + + @Override + public void writeStructBegin(TStruct struct) throws TException { + writeJSONObjectStart(); + } + + @Override + public void writeStructEnd() throws TException { + writeJSONObjectEnd(); + } + + @Override + public void writeFieldBegin(TField field) throws TException { + writeJSONInteger(field.id); + writeJSONObjectStart(); + writeJSONString(getTypeNameForTypeID(field.type)); + } + + @Override + public void writeFieldEnd() throws TException { + writeJSONObjectEnd(); + } + + @Override + public void writeFieldStop() {} + + @Override + public void writeMapBegin(TMap map) throws TException { + writeJSONArrayStart(); + writeJSONString(getTypeNameForTypeID(map.keyType)); + writeJSONString(getTypeNameForTypeID(map.valueType)); + writeJSONInteger(map.size); + writeJSONObjectStart(); + } + + @Override + public void writeMapEnd() throws TException { + writeJSONObjectEnd(); + writeJSONArrayEnd(); + } + + @Override + public void writeListBegin(TList list) throws TException { + writeJSONArrayStart(); + writeJSONString(getTypeNameForTypeID(list.elemType)); + writeJSONInteger(list.size); + } + + @Override + public void writeListEnd() throws TException { + writeJSONArrayEnd(); + } + + @Override + public void writeSetBegin(TSet set) throws TException { + writeJSONArrayStart(); + writeJSONString(getTypeNameForTypeID(set.elemType)); + writeJSONInteger(set.size); + } + + @Override + public void writeSetEnd() throws TException { + writeJSONArrayEnd(); + } + + @Override + public void writeBool(boolean b) throws TException { + writeJSONInteger(b ? (long)1 : (long)0); + } + + @Override + public void writeByte(byte b) throws TException { + writeJSONInteger((long)b); + } + + @Override + public void writeI16(short i16) throws TException { + writeJSONInteger((long)i16); + } + + @Override + public void writeI32(int i32) throws TException { + writeJSONInteger((long)i32); + } + + @Override + public void writeI64(long i64) throws TException { + writeJSONInteger(i64); + } + + @Override + public void writeDouble(double dub) throws TException { + writeJSONDouble(dub); + } + + @Override + public void writeString(String str) throws TException { + try { + byte[] b = str.getBytes("UTF-8"); + writeJSONString(b); + } catch (UnsupportedEncodingException uex) { + throw new TException("JVM DOES NOT SUPPORT UTF-8"); + } + } + + @Override + public void writeBinary(byte[] bin) throws TException { + writeJSONBase64(bin); + } + + /** + * Reading methods. + */ + + // Read in a JSON string, unescaping as appropriate.. Skip reading from the + // context if skipContext is true. + private TByteArrayOutputStream readJSONString(boolean skipContext) + throws TException { + TByteArrayOutputStream arr = new TByteArrayOutputStream(DEF_STRING_SIZE); + if (!skipContext) { + context_.read(); + } + readJSONSyntaxChar(QUOTE); + while (true) { + byte ch = reader_.read(); + if (ch == QUOTE[0]) { + break; + } + if (ch == ESCSEQ[0]) { + ch = reader_.read(); + if (ch == ESCSEQ[1]) { + readJSONSyntaxChar(ZERO); + readJSONSyntaxChar(ZERO); + trans_.readAll(tmpbuf_, 0, 2); + ch = (byte)((hexVal((byte)tmpbuf_[0]) << 4) + hexVal(tmpbuf_[1])); + } + else { + int off = ESCAPE_CHARS.indexOf(ch); + if (off == -1) { + throw new TProtocolException(TProtocolException.INVALID_DATA, + "Expected control char"); + } + ch = ESCAPE_CHAR_VALS[off]; + } + } + arr.write(ch); + } + return arr; + } + + // Return true if the given byte could be a valid part of a JSON number. + private boolean isJSONNumeric(byte b) { + switch (b) { + case '+': + case '-': + case '.': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 'E': + case 'e': + return true; + } + return false; + } + + // Read in a sequence of characters that are all valid in JSON numbers. Does + // not do a complete regex check to validate that this is actually a number. + private String readJSONNumericChars() throws TException { + StringBuilder strbld = new StringBuilder(); + while (true) { + byte ch = reader_.peek(); + if (!isJSONNumeric(ch)) { + break; + } + strbld.append((char)reader_.read()); + } + return strbld.toString(); + } + + // Read in a JSON number. If the context dictates, read in enclosing quotes. + private long readJSONInteger() throws TException { + context_.read(); + if (context_.escapeNum()) { + readJSONSyntaxChar(QUOTE); + } + String str = readJSONNumericChars(); + if (context_.escapeNum()) { + readJSONSyntaxChar(QUOTE); + } + try { + return Long.valueOf(str); + } + catch (NumberFormatException ex) { + throw new TProtocolException(TProtocolException.INVALID_DATA, + "Bad data encounted in numeric data"); + } + } + + // Read in a JSON double value. Throw if the value is not wrapped in quotes + // when expected or if wrapped in quotes when not expected. + private double readJSONDouble() throws TException { + context_.read(); + if (reader_.peek() == QUOTE[0]) { + TByteArrayOutputStream arr = readJSONString(true); + try { + double dub = Double.valueOf(arr.toString("UTF-8")); + if (!context_.escapeNum() && !Double.isNaN(dub) && + !Double.isInfinite(dub)) { + // Throw exception -- we should not be in a string in this case + throw new TProtocolException(TProtocolException.INVALID_DATA, + "Numeric data unexpectedly quoted"); + } + return dub; + } + catch (UnsupportedEncodingException ex) { + throw new TException("JVM DOES NOT SUPPORT UTF-8"); + } + } + else { + if (context_.escapeNum()) { + // This will throw - we should have had a quote if escapeNum == true + readJSONSyntaxChar(QUOTE); + } + try { + return Double.valueOf(readJSONNumericChars()); + } + catch (NumberFormatException ex) { + throw new TProtocolException(TProtocolException.INVALID_DATA, + "Bad data encounted in numeric data"); + } + } + } + + // Read in a JSON string containing base-64 encoded data and decode it. + private byte[] readJSONBase64() throws TException { + TByteArrayOutputStream arr = readJSONString(false); + byte[] b = arr.get(); + int len = arr.len(); + int off = 0; + int size = 0; + while (len >= 4) { + // Decode 4 bytes at a time + TBase64Utils.decode(b, off, 4, b, size); // NB: decoded in place + off += 4; + len -= 4; + size += 3; + } + // Don't decode if we hit the end or got a single leftover byte (invalid + // base64 but legal for skip of regular string type) + if (len > 1) { + // Decode remainder + TBase64Utils.decode(b, off, len, b, size); // NB: decoded in place + size += len - 1; + } + // Sadly we must copy the byte[] (any way around this?) + byte [] result = new byte[size]; + System.arraycopy(b, 0, result, 0, size); + return result; + } + + private void readJSONObjectStart() throws TException { + context_.read(); + readJSONSyntaxChar(LBRACE); + pushContext(new JSONPairContext()); + } + + private void readJSONObjectEnd() throws TException { + readJSONSyntaxChar(RBRACE); + popContext(); + } + + private void readJSONArrayStart() throws TException { + context_.read(); + readJSONSyntaxChar(LBRACKET); + pushContext(new JSONListContext()); + } + + private void readJSONArrayEnd() throws TException { + readJSONSyntaxChar(RBRACKET); + popContext(); + } + + @Override + public TMessage readMessageBegin() throws TException { + readJSONArrayStart(); + if (readJSONInteger() != VERSION) { + throw new TProtocolException(TProtocolException.BAD_VERSION, + "Message contained bad version."); + } + String name; + try { + name = readJSONString(false).toString("UTF-8"); + } + catch (UnsupportedEncodingException ex) { + throw new TException("JVM DOES NOT SUPPORT UTF-8"); + } + byte type = (byte) readJSONInteger(); + int seqid = (int) readJSONInteger(); + return new TMessage(name, type, seqid); + } + + @Override + public void readMessageEnd() throws TException { + readJSONArrayEnd(); + } + + @Override + public TStruct readStructBegin() throws TException { + readJSONObjectStart(); + return ANONYMOUS_STRUCT; + } + + @Override + public void readStructEnd() throws TException { + readJSONObjectEnd(); + } + + @Override + public TField readFieldBegin() throws TException { + byte ch = reader_.peek(); + byte type; + short id = 0; + if (ch == RBRACE[0]) { + type = TType.STOP; + } + else { + id = (short) readJSONInteger(); + readJSONObjectStart(); + type = getTypeIDForTypeName(readJSONString(false).get()); + } + return new TField("", type, id); + } + + @Override + public void readFieldEnd() throws TException { + readJSONObjectEnd(); + } + + @Override + public TMap readMapBegin() throws TException { + readJSONArrayStart(); + byte keyType = getTypeIDForTypeName(readJSONString(false).get()); + byte valueType = getTypeIDForTypeName(readJSONString(false).get()); + int size = (int)readJSONInteger(); + readJSONObjectStart(); + return new TMap(keyType, valueType, size); + } + + @Override + public void readMapEnd() throws TException { + readJSONObjectEnd(); + readJSONArrayEnd(); + } + + @Override + public TList readListBegin() throws TException { + readJSONArrayStart(); + byte elemType = getTypeIDForTypeName(readJSONString(false).get()); + int size = (int)readJSONInteger(); + return new TList(elemType, size); + } + + @Override + public void readListEnd() throws TException { + readJSONArrayEnd(); + } + + @Override + public TSet readSetBegin() throws TException { + readJSONArrayStart(); + byte elemType = getTypeIDForTypeName(readJSONString(false).get()); + int size = (int)readJSONInteger(); + return new TSet(elemType, size); + } + + @Override + public void readSetEnd() throws TException { + readJSONArrayEnd(); + } + + @Override + public boolean readBool() throws TException { + return (readJSONInteger() == 0 ? false : true); + } + + @Override + public byte readByte() throws TException { + return (byte) readJSONInteger(); + } + + @Override + public short readI16() throws TException { + return (short) readJSONInteger(); + } + + @Override + public int readI32() throws TException { + return (int) readJSONInteger(); + } + + @Override + public long readI64() throws TException { + return (long) readJSONInteger(); + } + + @Override + public double readDouble() throws TException { + return readJSONDouble(); + } + + @Override + public String readString() throws TException { + try { + return readJSONString(false).toString("UTF-8"); + } + catch (UnsupportedEncodingException ex) { + throw new TException("JVM DOES NOT SUPPORT UTF-8"); + } + } + + @Override + public byte[] readBinary() throws TException { + return readJSONBase64(); + } + +} diff --git a/lib/java/src/org/apache/thrift/protocol/TList.java b/lib/java/src/org/apache/thrift/protocol/TList.java new file mode 100644 index 00000000..0d36e83d --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TList.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** + * Helper class that encapsulates list metadata. + * + */ +public final class TList { + public TList() { + this(TType.STOP, 0); + } + + public TList(byte t, int s) { + elemType = t; + size = s; + } + + public final byte elemType; + public final int size; +} diff --git a/lib/java/src/org/apache/thrift/protocol/TMap.java b/lib/java/src/org/apache/thrift/protocol/TMap.java new file mode 100644 index 00000000..20881f7a --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TMap.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** + * Helper class that encapsulates map metadata. + * + */ +public final class TMap { + public TMap() { + this(TType.STOP, TType.STOP, 0); + } + + public TMap(byte k, byte v, int s) { + keyType = k; + valueType = v; + size = s; + } + + public final byte keyType; + public final byte valueType; + public final int size; +} diff --git a/lib/java/src/org/apache/thrift/protocol/TMessage.java b/lib/java/src/org/apache/thrift/protocol/TMessage.java new file mode 100644 index 00000000..cd56964d --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TMessage.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** + * Helper class that encapsulates struct metadata. + * + */ +public final class TMessage { + public TMessage() { + this("", TType.STOP, 0); + } + + public TMessage(String n, byte t, int s) { + name = n; + type = t; + seqid = s; + } + + public final String name; + public final byte type; + public final int seqid; + + public String toString() { + return ""; + } + + public boolean equals(TMessage other) { + return name.equals(other.name) && type == other.type && seqid == other.seqid; + } +} diff --git a/lib/java/src/org/apache/thrift/protocol/TMessageType.java b/lib/java/src/org/apache/thrift/protocol/TMessageType.java new file mode 100644 index 00000000..aa3f9317 --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TMessageType.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** + * Message type constants in the Thrift protocol. + * + */ +public final class TMessageType { + public static final byte CALL = 1; + public static final byte REPLY = 2; + public static final byte EXCEPTION = 3; + public static final byte ONEWAY = 4; +} diff --git a/lib/java/src/org/apache/thrift/protocol/TProtocol.java b/lib/java/src/org/apache/thrift/protocol/TProtocol.java new file mode 100644 index 00000000..50d6683d --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TProtocol.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransport; + +/** + * Protocol interface definition. + * + */ +public abstract class TProtocol { + + /** + * Prevent direct instantiation + */ + @SuppressWarnings("unused") + private TProtocol() {} + + /** + * Transport + */ + protected TTransport trans_; + + /** + * Constructor + */ + protected TProtocol(TTransport trans) { + trans_ = trans; + } + + /** + * Transport accessor + */ + public TTransport getTransport() { + return trans_; + } + + /** + * Writing methods. + */ + + public abstract void writeMessageBegin(TMessage message) throws TException; + + public abstract void writeMessageEnd() throws TException; + + public abstract void writeStructBegin(TStruct struct) throws TException; + + public abstract void writeStructEnd() throws TException; + + public abstract void writeFieldBegin(TField field) throws TException; + + public abstract void writeFieldEnd() throws TException; + + public abstract void writeFieldStop() throws TException; + + public abstract void writeMapBegin(TMap map) throws TException; + + public abstract void writeMapEnd() throws TException; + + public abstract void writeListBegin(TList list) throws TException; + + public abstract void writeListEnd() throws TException; + + public abstract void writeSetBegin(TSet set) throws TException; + + public abstract void writeSetEnd() throws TException; + + public abstract void writeBool(boolean b) throws TException; + + public abstract void writeByte(byte b) throws TException; + + public abstract void writeI16(short i16) throws TException; + + public abstract void writeI32(int i32) throws TException; + + public abstract void writeI64(long i64) throws TException; + + public abstract void writeDouble(double dub) throws TException; + + public abstract void writeString(String str) throws TException; + + public abstract void writeBinary(byte[] bin) throws TException; + + /** + * Reading methods. + */ + + public abstract TMessage readMessageBegin() throws TException; + + public abstract void readMessageEnd() throws TException; + + public abstract TStruct readStructBegin() throws TException; + + public abstract void readStructEnd() throws TException; + + public abstract TField readFieldBegin() throws TException; + + public abstract void readFieldEnd() throws TException; + + public abstract TMap readMapBegin() throws TException; + + public abstract void readMapEnd() throws TException; + + public abstract TList readListBegin() throws TException; + + public abstract void readListEnd() throws TException; + + public abstract TSet readSetBegin() throws TException; + + public abstract void readSetEnd() throws TException; + + public abstract boolean readBool() throws TException; + + public abstract byte readByte() throws TException; + + public abstract short readI16() throws TException; + + public abstract int readI32() throws TException; + + public abstract long readI64() throws TException; + + public abstract double readDouble() throws TException; + + public abstract String readString() throws TException; + + public abstract byte[] readBinary() throws TException; + +} diff --git a/lib/java/src/org/apache/thrift/protocol/TProtocolException.java b/lib/java/src/org/apache/thrift/protocol/TProtocolException.java new file mode 100644 index 00000000..248815be --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TProtocolException.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import org.apache.thrift.TException; + +/** + * Protocol exceptions. + * + */ +public class TProtocolException extends TException { + + + private static final long serialVersionUID = 1L; + public static final int UNKNOWN = 0; + public static final int INVALID_DATA = 1; + public static final int NEGATIVE_SIZE = 2; + public static final int SIZE_LIMIT = 3; + public static final int BAD_VERSION = 4; + public static final int NOT_IMPLEMENTED = 5; + + protected int type_ = UNKNOWN; + + public TProtocolException() { + super(); + } + + public TProtocolException(int type) { + super(); + type_ = type; + } + + public TProtocolException(int type, String message) { + super(message); + type_ = type; + } + + public TProtocolException(String message) { + super(message); + } + + public TProtocolException(int type, Throwable cause) { + super(cause); + type_ = type; + } + + public TProtocolException(Throwable cause) { + super(cause); + } + + public TProtocolException(String message, Throwable cause) { + super(message, cause); + } + + public TProtocolException(int type, String message, Throwable cause) { + super(message, cause); + type_ = type; + } + + public int getType() { + return type_; + } + +} diff --git a/lib/java/src/org/apache/thrift/protocol/TProtocolFactory.java b/lib/java/src/org/apache/thrift/protocol/TProtocolFactory.java new file mode 100644 index 00000000..afa502b7 --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TProtocolFactory.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import org.apache.thrift.transport.TTransport; + +/** + * Factory interface for constructing protocol instances. + * + */ +public interface TProtocolFactory { + public TProtocol getProtocol(TTransport trans); +} diff --git a/lib/java/src/org/apache/thrift/protocol/TProtocolUtil.java b/lib/java/src/org/apache/thrift/protocol/TProtocolUtil.java new file mode 100644 index 00000000..9bf10f67 --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TProtocolUtil.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import org.apache.thrift.TException; + +/** + * Utility class with static methods for interacting with protocol data + * streams. + * + */ +public class TProtocolUtil { + + /** + * The maximum recursive depth the skip() function will traverse before + * throwing a TException. + */ + private static int maxSkipDepth = Integer.MAX_VALUE; + + /** + * Specifies the maximum recursive depth that the skip function will + * traverse before throwing a TException. This is a global setting, so + * any call to skip in this JVM will enforce this value. + * + * @param depth the maximum recursive depth. A value of 2 would allow + * the skip function to skip a structure or collection with basic children, + * but it would not permit skipping a struct that had a field containing + * a child struct. A value of 1 would only allow skipping of simple + * types and empty structs/collections. + */ + public static void setMaxSkipDepth(int depth) { + maxSkipDepth = depth; + } + + /** + * Skips over the next data element from the provided input TProtocol object. + * + * @param prot the protocol object to read from + * @param type the next value will be intepreted as this TType value. + */ + public static void skip(TProtocol prot, byte type) + throws TException { + skip(prot, type, maxSkipDepth); + } + + /** + * Skips over the next data element from the provided input TProtocol object. + * + * @param prot the protocol object to read from + * @param type the next value will be intepreted as this TType value. + * @param maxDepth this function will only skip complex objects to this + * recursive depth, to prevent Java stack overflow. + */ + public static void skip(TProtocol prot, byte type, int maxDepth) + throws TException { + if (maxDepth <= 0) { + throw new TException("Maximum skip depth exceeded"); + } + switch (type) { + case TType.BOOL: + { + prot.readBool(); + break; + } + case TType.BYTE: + { + prot.readByte(); + break; + } + case TType.I16: + { + prot.readI16(); + break; + } + case TType.I32: + { + prot.readI32(); + break; + } + case TType.I64: + { + prot.readI64(); + break; + } + case TType.DOUBLE: + { + prot.readDouble(); + break; + } + case TType.STRING: + { + prot.readBinary(); + break; + } + case TType.STRUCT: + { + prot.readStructBegin(); + while (true) { + TField field = prot.readFieldBegin(); + if (field.type == TType.STOP) { + break; + } + skip(prot, field.type, maxDepth - 1); + prot.readFieldEnd(); + } + prot.readStructEnd(); + break; + } + case TType.MAP: + { + TMap map = prot.readMapBegin(); + for (int i = 0; i < map.size; i++) { + skip(prot, map.keyType, maxDepth - 1); + skip(prot, map.valueType, maxDepth - 1); + } + prot.readMapEnd(); + break; + } + case TType.SET: + { + TSet set = prot.readSetBegin(); + for (int i = 0; i < set.size; i++) { + skip(prot, set.elemType, maxDepth - 1); + } + prot.readSetEnd(); + break; + } + case TType.LIST: + { + TList list = prot.readListBegin(); + for (int i = 0; i < list.size; i++) { + skip(prot, list.elemType, maxDepth - 1); + } + prot.readListEnd(); + break; + } + default: + break; + } + } +} diff --git a/lib/java/src/org/apache/thrift/protocol/TSet.java b/lib/java/src/org/apache/thrift/protocol/TSet.java new file mode 100644 index 00000000..38be9a99 --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TSet.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** + * Helper class that encapsulates set metadata. + * + */ +public final class TSet { + public TSet() { + this(TType.STOP, 0); + } + + public TSet(byte t, int s) { + elemType = t; + size = s; + } + + public TSet(TList list) { + this(list.elemType, list.size); + } + + public final byte elemType; + public final int size; +} diff --git a/lib/java/src/org/apache/thrift/protocol/TSimpleJSONProtocol.java b/lib/java/src/org/apache/thrift/protocol/TSimpleJSONProtocol.java new file mode 100644 index 00000000..a60bdf40 --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TSimpleJSONProtocol.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import java.io.UnsupportedEncodingException; +import java.util.Stack; + +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransport; + +/** + * JSON protocol implementation for thrift. + * + * This protocol is write-only and produces a simple output format + * suitable for parsing by scripting languages. It should not be + * confused with the full-featured TJSONProtocol. + * + */ +public class TSimpleJSONProtocol extends TProtocol { + + /** + * Factory + */ + public static class Factory implements TProtocolFactory { + public TProtocol getProtocol(TTransport trans) { + return new TSimpleJSONProtocol(trans); + } + } + + public static final byte[] COMMA = new byte[] {','}; + public static final byte[] COLON = new byte[] {':'}; + public static final byte[] LBRACE = new byte[] {'{'}; + public static final byte[] RBRACE = new byte[] {'}'}; + public static final byte[] LBRACKET = new byte[] {'['}; + public static final byte[] RBRACKET = new byte[] {']'}; + public static final char QUOTE = '"'; + + private static final TStruct ANONYMOUS_STRUCT = new TStruct(); + private static final TField ANONYMOUS_FIELD = new TField(); + private static final TMessage EMPTY_MESSAGE = new TMessage(); + private static final TSet EMPTY_SET = new TSet(); + private static final TList EMPTY_LIST = new TList(); + private static final TMap EMPTY_MAP = new TMap(); + + protected class Context { + protected void write() throws TException {} + } + + protected class ListContext extends Context { + protected boolean first_ = true; + + protected void write() throws TException { + if (first_) { + first_ = false; + } else { + trans_.write(COMMA); + } + } + } + + protected class StructContext extends Context { + protected boolean first_ = true; + protected boolean colon_ = true; + + protected void write() throws TException { + if (first_) { + first_ = false; + colon_ = true; + } else { + trans_.write(colon_ ? COLON : COMMA); + colon_ = !colon_; + } + } + } + + protected final Context BASE_CONTEXT = new Context(); + + /** + * Stack of nested contexts that we may be in. + */ + protected Stack writeContextStack_ = new Stack(); + + /** + * Current context that we are in + */ + protected Context writeContext_ = BASE_CONTEXT; + + /** + * Push a new write context onto the stack. + */ + protected void pushWriteContext(Context c) { + writeContextStack_.push(writeContext_); + writeContext_ = c; + } + + /** + * Pop the last write context off the stack + */ + protected void popWriteContext() { + writeContext_ = writeContextStack_.pop(); + } + + /** + * Constructor + */ + public TSimpleJSONProtocol(TTransport trans) { + super(trans); + } + + public void writeMessageBegin(TMessage message) throws TException { + trans_.write(LBRACKET); + pushWriteContext(new ListContext()); + writeString(message.name); + writeByte(message.type); + writeI32(message.seqid); + } + + public void writeMessageEnd() throws TException { + popWriteContext(); + trans_.write(RBRACKET); + } + + public void writeStructBegin(TStruct struct) throws TException { + writeContext_.write(); + trans_.write(LBRACE); + pushWriteContext(new StructContext()); + } + + public void writeStructEnd() throws TException { + popWriteContext(); + trans_.write(RBRACE); + } + + public void writeFieldBegin(TField field) throws TException { + // Note that extra type information is omitted in JSON! + writeString(field.name); + } + + public void writeFieldEnd() {} + + public void writeFieldStop() {} + + public void writeMapBegin(TMap map) throws TException { + writeContext_.write(); + trans_.write(LBRACE); + pushWriteContext(new StructContext()); + // No metadata! + } + + public void writeMapEnd() throws TException { + popWriteContext(); + trans_.write(RBRACE); + } + + public void writeListBegin(TList list) throws TException { + writeContext_.write(); + trans_.write(LBRACKET); + pushWriteContext(new ListContext()); + // No metadata! + } + + public void writeListEnd() throws TException { + popWriteContext(); + trans_.write(RBRACKET); + } + + public void writeSetBegin(TSet set) throws TException { + writeContext_.write(); + trans_.write(LBRACKET); + pushWriteContext(new ListContext()); + // No metadata! + } + + public void writeSetEnd() throws TException { + popWriteContext(); + trans_.write(RBRACKET); + } + + public void writeBool(boolean b) throws TException { + writeByte(b ? (byte)1 : (byte)0); + } + + public void writeByte(byte b) throws TException { + writeI32(b); + } + + public void writeI16(short i16) throws TException { + writeI32(i16); + } + + public void writeI32(int i32) throws TException { + writeContext_.write(); + _writeStringData(Integer.toString(i32)); + } + + public void _writeStringData(String s) throws TException { + try { + byte[] b = s.getBytes("UTF-8"); + trans_.write(b); + } catch (UnsupportedEncodingException uex) { + throw new TException("JVM DOES NOT SUPPORT UTF-8"); + } + } + + public void writeI64(long i64) throws TException { + writeContext_.write(); + _writeStringData(Long.toString(i64)); + } + + public void writeDouble(double dub) throws TException { + writeContext_.write(); + _writeStringData(Double.toString(dub)); + } + + public void writeString(String str) throws TException { + writeContext_.write(); + int length = str.length(); + StringBuffer escape = new StringBuffer(length + 16); + escape.append(QUOTE); + for (int i = 0; i < length; ++i) { + char c = str.charAt(i); + switch (c) { + case '"': + case '\\': + escape.append('\\'); + escape.append(c); + break; + case '\b': + escape.append('\\'); + escape.append('b'); + break; + case '\f': + escape.append('\\'); + escape.append('f'); + break; + case '\n': + escape.append('\\'); + escape.append('n'); + break; + case '\r': + escape.append('\\'); + escape.append('r'); + break; + case '\t': + escape.append('\\'); + escape.append('t'); + break; + default: + // Control characeters! According to JSON RFC u0020 (space) + if (c < ' ') { + String hex = Integer.toHexString(c); + escape.append('\\'); + escape.append('u'); + for (int j = 4; j > hex.length(); --j) { + escape.append('0'); + } + escape.append(hex); + } else { + escape.append(c); + } + break; + } + } + escape.append(QUOTE); + _writeStringData(escape.toString()); + } + + public void writeBinary(byte[] bin) throws TException { + try { + // TODO(mcslee): Fix this + writeString(new String(bin, "UTF-8")); + } catch (UnsupportedEncodingException uex) { + throw new TException("JVM DOES NOT SUPPORT UTF-8"); + } + } + + /** + * Reading methods. + */ + + public TMessage readMessageBegin() throws TException { + // TODO(mcslee): implement + return EMPTY_MESSAGE; + } + + public void readMessageEnd() {} + + public TStruct readStructBegin() { + // TODO(mcslee): implement + return ANONYMOUS_STRUCT; + } + + public void readStructEnd() {} + + public TField readFieldBegin() throws TException { + // TODO(mcslee): implement + return ANONYMOUS_FIELD; + } + + public void readFieldEnd() {} + + public TMap readMapBegin() throws TException { + // TODO(mcslee): implement + return EMPTY_MAP; + } + + public void readMapEnd() {} + + public TList readListBegin() throws TException { + // TODO(mcslee): implement + return EMPTY_LIST; + } + + public void readListEnd() {} + + public TSet readSetBegin() throws TException { + // TODO(mcslee): implement + return EMPTY_SET; + } + + public void readSetEnd() {} + + public boolean readBool() throws TException { + return (readByte() == 1); + } + + public byte readByte() throws TException { + // TODO(mcslee): implement + return 0; + } + + public short readI16() throws TException { + // TODO(mcslee): implement + return 0; + } + + public int readI32() throws TException { + // TODO(mcslee): implement + return 0; + } + + public long readI64() throws TException { + // TODO(mcslee): implement + return 0; + } + + public double readDouble() throws TException { + // TODO(mcslee): implement + return 0; + } + + public String readString() throws TException { + // TODO(mcslee): implement + return ""; + } + + public String readStringBody(int size) throws TException { + // TODO(mcslee): implement + return ""; + } + + public byte[] readBinary() throws TException { + // TODO(mcslee): implement + return new byte[0]; + } + +} diff --git a/lib/java/src/org/apache/thrift/protocol/TStruct.java b/lib/java/src/org/apache/thrift/protocol/TStruct.java new file mode 100644 index 00000000..a0f79012 --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TStruct.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** + * Helper class that encapsulates struct metadata. + * + */ +public final class TStruct { + public TStruct() { + this(""); + } + + public TStruct(String n) { + name = n; + } + + public final String name; +} diff --git a/lib/java/src/org/apache/thrift/protocol/TType.java b/lib/java/src/org/apache/thrift/protocol/TType.java new file mode 100644 index 00000000..dbdc3caa --- /dev/null +++ b/lib/java/src/org/apache/thrift/protocol/TType.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** + * Type constants in the Thrift protocol. + * + */ +public final class TType { + public static final byte STOP = 0; + public static final byte VOID = 1; + public static final byte BOOL = 2; + public static final byte BYTE = 3; + public static final byte DOUBLE = 4; + public static final byte I16 = 6; + public static final byte I32 = 8; + public static final byte I64 = 10; + public static final byte STRING = 11; + public static final byte STRUCT = 12; + public static final byte MAP = 13; + public static final byte SET = 14; + public static final byte LIST = 15; +} diff --git a/lib/java/src/org/apache/thrift/server/THsHaServer.java b/lib/java/src/org/apache/thrift/server/THsHaServer.java new file mode 100644 index 00000000..8bf096ed --- /dev/null +++ b/lib/java/src/org/apache/thrift/server/THsHaServer.java @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.thrift.server; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.thrift.TProcessor; +import org.apache.thrift.TProcessorFactory; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TNonblockingServerTransport; + +/** + * An extension of the TNonblockingServer to a Half-Sync/Half-Async server. + * Like TNonblockingServer, it relies on the use of TFramedTransport. + */ +public class THsHaServer extends TNonblockingServer { + + // This wraps all the functionality of queueing and thread pool management + // for the passing of Invocations from the Selector to workers. + private ExecutorService invoker; + + protected final int MIN_WORKER_THREADS; + protected final int MAX_WORKER_THREADS; + protected final int STOP_TIMEOUT_VAL; + protected final TimeUnit STOP_TIMEOUT_UNIT; + + /** + * Create server with given processor, and server transport. Default server + * options, TBinaryProtocol for the protocol, and TFramedTransport.Factory on + * both input and output transports. A TProcessorFactory will be created that + * always returns the specified processor. + */ + public THsHaServer( TProcessor processor, + TNonblockingServerTransport serverTransport) { + this(processor, serverTransport, new Options()); + } + + /** + * Create server with given processor, server transport, and server options + * using TBinaryProtocol for the protocol, and TFramedTransport.Factory on + * both input and output transports. A TProcessorFactory will be created that + * always returns the specified processor. + */ + public THsHaServer( TProcessor processor, + TNonblockingServerTransport serverTransport, + Options options) { + this(new TProcessorFactory(processor), serverTransport, options); + } + + /** + * Create server with specified processor factory and server transport. Uses + * default options. TBinaryProtocol is assumed. TFramedTransport.Factory is + * used on both input and output transports. + */ + public THsHaServer( TProcessorFactory processorFactory, + TNonblockingServerTransport serverTransport) { + this(processorFactory, serverTransport, new Options()); + } + + /** + * Create server with specified processor factory, server transport, and server + * options. TBinaryProtocol is assumed. TFramedTransport.Factory is used on + * both input and output transports. + */ + public THsHaServer( TProcessorFactory processorFactory, + TNonblockingServerTransport serverTransport, + Options options) { + this(processorFactory, serverTransport, new TFramedTransport.Factory(), + new TBinaryProtocol.Factory(), options); + } + + /** + * Server with specified processor, server transport, and in/out protocol + * factory. Defaults will be used for in/out transport factory and server + * options. + */ + public THsHaServer( TProcessor processor, + TNonblockingServerTransport serverTransport, + TProtocolFactory protocolFactory) { + this(processor, serverTransport, protocolFactory, new Options()); + } + + /** + * Server with specified processor, server transport, and in/out protocol + * factory. Defaults will be used for in/out transport factory and server + * options. + */ + public THsHaServer( TProcessor processor, + TNonblockingServerTransport serverTransport, + TProtocolFactory protocolFactory, + Options options) { + this(processor, serverTransport, new TFramedTransport.Factory(), + protocolFactory); + } + + /** + * Create server with specified processor, server transport, in/out + * transport factory, in/out protocol factory, and default server options. A + * processor factory will be created that always returns the specified + * processor. + */ + public THsHaServer( TProcessor processor, + TNonblockingServerTransport serverTransport, + TFramedTransport.Factory transportFactory, + TProtocolFactory protocolFactory) { + this(new TProcessorFactory(processor), serverTransport, + transportFactory, protocolFactory); + } + + /** + * Create server with specified processor factory, server transport, in/out + * transport factory, in/out protocol factory, and default server options. + */ + public THsHaServer( TProcessorFactory processorFactory, + TNonblockingServerTransport serverTransport, + TFramedTransport.Factory transportFactory, + TProtocolFactory protocolFactory) { + this(processorFactory, serverTransport, + transportFactory, transportFactory, + protocolFactory, protocolFactory, new Options()); + } + + /** + * Create server with specified processor factory, server transport, in/out + * transport factory, in/out protocol factory, and server options. + */ + public THsHaServer( TProcessorFactory processorFactory, + TNonblockingServerTransport serverTransport, + TFramedTransport.Factory transportFactory, + TProtocolFactory protocolFactory, + Options options) { + this(processorFactory, serverTransport, + transportFactory, transportFactory, + protocolFactory, protocolFactory, + options); + } + + /** + * Create server with everything specified, except use default server options. + */ + public THsHaServer( TProcessor processor, + TNonblockingServerTransport serverTransport, + TFramedTransport.Factory inputTransportFactory, + TFramedTransport.Factory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory) { + this(new TProcessorFactory(processor), serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory); + } + + /** + * Create server with everything specified, except use default server options. + */ + public THsHaServer( TProcessorFactory processorFactory, + TNonblockingServerTransport serverTransport, + TFramedTransport.Factory inputTransportFactory, + TFramedTransport.Factory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory) + { + this(processorFactory, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory, new Options()); + } + + /** + * Create server with every option fully specified. + */ + public THsHaServer( TProcessorFactory processorFactory, + TNonblockingServerTransport serverTransport, + TFramedTransport.Factory inputTransportFactory, + TFramedTransport.Factory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory, + Options options) + { + super(processorFactory, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory, + options); + + MIN_WORKER_THREADS = options.minWorkerThreads; + MAX_WORKER_THREADS = options.maxWorkerThreads; + STOP_TIMEOUT_VAL = options.stopTimeoutVal; + STOP_TIMEOUT_UNIT = options.stopTimeoutUnit; + } + + /** @inheritDoc */ + @Override + public void serve() { + if (!startInvokerPool()) { + return; + } + + // start listening, or exit + if (!startListening()) { + return; + } + + // start the selector, or exit + if (!startSelectorThread()) { + return; + } + + // this will block while we serve + joinSelector(); + + gracefullyShutdownInvokerPool(); + + // do a little cleanup + stopListening(); + + // ungracefully shut down the invoker pool? + } + + protected boolean startInvokerPool() { + // start the invoker pool + LinkedBlockingQueue queue = new LinkedBlockingQueue(); + invoker = new ThreadPoolExecutor(MIN_WORKER_THREADS, MAX_WORKER_THREADS, + STOP_TIMEOUT_VAL, STOP_TIMEOUT_UNIT, queue); + + return true; + } + + protected void gracefullyShutdownInvokerPool() { + // try to gracefully shut down the executor service + invoker.shutdown(); + + // Loop until awaitTermination finally does return without a interrupted + // exception. If we don't do this, then we'll shut down prematurely. We want + // to let the executorService clear it's task queue, closing client sockets + // appropriately. + long timeoutMS = 10000; + long now = System.currentTimeMillis(); + while (timeoutMS >= 0) { + try { + invoker.awaitTermination(timeoutMS, TimeUnit.MILLISECONDS); + break; + } catch (InterruptedException ix) { + long newnow = System.currentTimeMillis(); + timeoutMS -= (newnow - now); + now = newnow; + } + } + } + + /** + * We override the standard invoke method here to queue the invocation for + * invoker service instead of immediately invoking. The thread pool takes care of the rest. + */ + @Override + protected void requestInvoke(FrameBuffer frameBuffer) { + invoker.execute(new Invocation(frameBuffer)); + } + + /** + * An Invocation represents a method call that is prepared to execute, given + * an idle worker thread. It contains the input and output protocols the + * thread's processor should use to perform the usual Thrift invocation. + */ + private class Invocation implements Runnable { + + private final FrameBuffer frameBuffer; + + public Invocation(final FrameBuffer frameBuffer) { + this.frameBuffer = frameBuffer; + } + + public void run() { + frameBuffer.invoke(); + } + } + + public static class Options extends TNonblockingServer.Options { + public int minWorkerThreads = 5; + public int maxWorkerThreads = Integer.MAX_VALUE; + public int stopTimeoutVal = 60; + public TimeUnit stopTimeoutUnit = TimeUnit.SECONDS; + } +} diff --git a/lib/java/src/org/apache/thrift/server/TNonblockingServer.java b/lib/java/src/org/apache/thrift/server/TNonblockingServer.java new file mode 100644 index 00000000..95d81e22 --- /dev/null +++ b/lib/java/src/org/apache/thrift/server/TNonblockingServer.java @@ -0,0 +1,769 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.thrift.server; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.spi.SelectorProvider; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import org.apache.log4j.Logger; + +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; +import org.apache.thrift.TProcessorFactory; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.thrift.transport.TNonblockingServerTransport; +import org.apache.thrift.transport.TNonblockingTransport; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +/** + * A nonblocking TServer implementation. This allows for fairness amongst all + * connected clients in terms of invocations. + * + * This server is inherently single-threaded. If you want a limited thread pool + * coupled with invocation-fairness, see THsHaServer. + * + * To use this server, you MUST use a TFramedTransport at the outermost + * transport, otherwise this server will be unable to determine when a whole + * method call has been read off the wire. Clients must also use TFramedTransport. + */ +public class TNonblockingServer extends TServer { + private static final Logger LOGGER = + Logger.getLogger(TNonblockingServer.class.getName()); + + // Flag for stopping the server + private volatile boolean stopped_; + + private SelectThread selectThread_; + + /** + * The maximum amount of memory we will allocate to client IO buffers at a + * time. Without this limit, the server will gladly allocate client buffers + * right into an out of memory exception, rather than waiting. + */ + private final long MAX_READ_BUFFER_BYTES; + + protected final Options options_; + + /** + * How many bytes are currently allocated to read buffers. + */ + private long readBufferBytesAllocated = 0; + + /** + * Create server with given processor and server transport, using + * TBinaryProtocol for the protocol, TFramedTransport.Factory on both input + * and output transports. A TProcessorFactory will be created that always + * returns the specified processor. + */ + public TNonblockingServer(TProcessor processor, + TNonblockingServerTransport serverTransport) { + this(new TProcessorFactory(processor), serverTransport); + } + + /** + * Create server with specified processor factory and server transport. + * TBinaryProtocol is assumed. TFramedTransport.Factory is used on both input + * and output transports. + */ + public TNonblockingServer(TProcessorFactory processorFactory, + TNonblockingServerTransport serverTransport) { + this(processorFactory, serverTransport, + new TFramedTransport.Factory(), new TFramedTransport.Factory(), + new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory()); + } + + public TNonblockingServer(TProcessor processor, + TNonblockingServerTransport serverTransport, + TProtocolFactory protocolFactory) { + this(processor, serverTransport, + new TFramedTransport.Factory(), new TFramedTransport.Factory(), + protocolFactory, protocolFactory); + } + + public TNonblockingServer(TProcessor processor, + TNonblockingServerTransport serverTransport, + TFramedTransport.Factory transportFactory, + TProtocolFactory protocolFactory) { + this(processor, serverTransport, + transportFactory, transportFactory, + protocolFactory, protocolFactory); + } + + public TNonblockingServer(TProcessorFactory processorFactory, + TNonblockingServerTransport serverTransport, + TFramedTransport.Factory transportFactory, + TProtocolFactory protocolFactory) { + this(processorFactory, serverTransport, + transportFactory, transportFactory, + protocolFactory, protocolFactory); + } + + public TNonblockingServer(TProcessor processor, + TNonblockingServerTransport serverTransport, + TFramedTransport.Factory inputTransportFactory, + TFramedTransport.Factory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory) { + this(new TProcessorFactory(processor), serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory); + } + + public TNonblockingServer(TProcessorFactory processorFactory, + TNonblockingServerTransport serverTransport, + TFramedTransport.Factory inputTransportFactory, + TFramedTransport.Factory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory) { + this(processorFactory, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory, + new Options()); + } + + public TNonblockingServer(TProcessorFactory processorFactory, + TNonblockingServerTransport serverTransport, + TFramedTransport.Factory inputTransportFactory, + TFramedTransport.Factory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory, + Options options) { + super(processorFactory, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory); + options_ = options; + options_.validate(); + MAX_READ_BUFFER_BYTES = options.maxReadBufferBytes; + } + + /** + * Begin accepting connections and processing invocations. + */ + public void serve() { + // start listening, or exit + if (!startListening()) { + return; + } + + // start the selector, or exit + if (!startSelectorThread()) { + return; + } + + // this will block while we serve + joinSelector(); + + // do a little cleanup + stopListening(); + } + + /** + * Have the server transport start accepting connections. + * + * @return true if we started listening successfully, false if something went + * wrong. + */ + protected boolean startListening() { + try { + serverTransport_.listen(); + return true; + } catch (TTransportException ttx) { + LOGGER.error("Failed to start listening on server socket!", ttx); + return false; + } + } + + /** + * Stop listening for conections. + */ + protected void stopListening() { + serverTransport_.close(); + } + + /** + * Start the selector thread running to deal with clients. + * + * @return true if everything went ok, false if we couldn't start for some + * reason. + */ + protected boolean startSelectorThread() { + // start the selector + try { + selectThread_ = new SelectThread((TNonblockingServerTransport)serverTransport_); + selectThread_.start(); + return true; + } catch (IOException e) { + LOGGER.error("Failed to start selector thread!", e); + return false; + } + } + + /** + * Block until the selector exits. + */ + protected void joinSelector() { + // wait until the selector thread exits + try { + selectThread_.join(); + } catch (InterruptedException e) { + // for now, just silently ignore. technically this means we'll have less of + // a graceful shutdown as a result. + } + } + + /** + * Stop serving and shut everything down. + */ + public void stop() { + stopped_ = true; + selectThread_.wakeupSelector(); + } + + /** + * Perform an invocation. This method could behave several different ways + * - invoke immediately inline, queue for separate execution, etc. + */ + protected void requestInvoke(FrameBuffer frameBuffer) { + frameBuffer.invoke(); + } + + /** + * A FrameBuffer wants to change its selection preferences, but might not be + * in the select thread. + */ + protected void requestSelectInterestChange(FrameBuffer frameBuffer) { + selectThread_.requestSelectInterestChange(frameBuffer); + } + + /** + * The thread that will be doing all the selecting, managing new connections + * and those that still need to be read. + */ + protected class SelectThread extends Thread { + + private final TNonblockingServerTransport serverTransport; + private final Selector selector; + + // List of FrameBuffers that want to change their selection interests. + private final Set selectInterestChanges = + new HashSet(); + + /** + * Set up the SelectorThread. + */ + public SelectThread(final TNonblockingServerTransport serverTransport) + throws IOException { + this.serverTransport = serverTransport; + this.selector = SelectorProvider.provider().openSelector(); + serverTransport.registerSelector(selector); + } + + /** + * The work loop. Handles both selecting (all IO operations) and managing + * the selection preferences of all existing connections. + */ + public void run() { + while (!stopped_) { + select(); + processInterestChanges(); + } + } + + /** + * If the selector is blocked, wake it up. + */ + public void wakeupSelector() { + selector.wakeup(); + } + + /** + * Add FrameBuffer to the list of select interest changes and wake up the + * selector if it's blocked. When the select() call exits, it'll give the + * FrameBuffer a chance to change its interests. + */ + public void requestSelectInterestChange(FrameBuffer frameBuffer) { + synchronized (selectInterestChanges) { + selectInterestChanges.add(frameBuffer); + } + // wakeup the selector, if it's currently blocked. + selector.wakeup(); + } + + /** + * Select and process IO events appropriately: + * If there are connections to be accepted, accept them. + * If there are existing connections with data waiting to be read, read it, + * bufferring until a whole frame has been read. + * If there are any pending responses, buffer them until their target client + * is available, and then send the data. + */ + private void select() { + try { + // wait for io events. + selector.select(); + + // process the io events we received + Iterator selectedKeys = selector.selectedKeys().iterator(); + while (!stopped_ && selectedKeys.hasNext()) { + SelectionKey key = selectedKeys.next(); + selectedKeys.remove(); + + // skip if not valid + if (!key.isValid()) { + cleanupSelectionkey(key); + continue; + } + + // if the key is marked Accept, then it has to be the server + // transport. + if (key.isAcceptable()) { + handleAccept(); + } else if (key.isReadable()) { + // deal with reads + handleRead(key); + } else if (key.isWritable()) { + // deal with writes + handleWrite(key); + } else { + LOGGER.warn("Unexpected state in select! " + key.interestOps()); + } + } + } catch (IOException e) { + LOGGER.warn("Got an IOException while selecting!", e); + } + } + + /** + * Check to see if there are any FrameBuffers that have switched their + * interest type from read to write or vice versa. + */ + private void processInterestChanges() { + synchronized (selectInterestChanges) { + for (FrameBuffer fb : selectInterestChanges) { + fb.changeSelectInterests(); + } + selectInterestChanges.clear(); + } + } + + /** + * Accept a new connection. + */ + private void handleAccept() throws IOException { + SelectionKey clientKey = null; + TNonblockingTransport client = null; + try { + // accept the connection + client = (TNonblockingTransport)serverTransport.accept(); + clientKey = client.registerSelector(selector, SelectionKey.OP_READ); + + // add this key to the map + FrameBuffer frameBuffer = new FrameBuffer(client, clientKey); + clientKey.attach(frameBuffer); + } catch (TTransportException tte) { + // something went wrong accepting. + LOGGER.warn("Exception trying to accept!", tte); + tte.printStackTrace(); + if (clientKey != null) cleanupSelectionkey(clientKey); + if (client != null) client.close(); + } + } + + /** + * Do the work required to read from a readable client. If the frame is + * fully read, then invoke the method call. + */ + private void handleRead(SelectionKey key) { + FrameBuffer buffer = (FrameBuffer)key.attachment(); + if (buffer.read()) { + // if the buffer's frame read is complete, invoke the method. + if (buffer.isFrameFullyRead()) { + requestInvoke(buffer); + } + } else { + cleanupSelectionkey(key); + } + } + + /** + * Let a writable client get written, if there's data to be written. + */ + private void handleWrite(SelectionKey key) { + FrameBuffer buffer = (FrameBuffer)key.attachment(); + if (!buffer.write()) { + cleanupSelectionkey(key); + } + } + + /** + * Do connection-close cleanup on a given SelectionKey. + */ + private void cleanupSelectionkey(SelectionKey key) { + // remove the records from the two maps + FrameBuffer buffer = (FrameBuffer)key.attachment(); + if (buffer != null) { + // close the buffer + buffer.close(); + } + // cancel the selection key + key.cancel(); + } + } // SelectorThread + + /** + * Class that implements a sort of state machine around the interaction with + * a client and an invoker. It manages reading the frame size and frame data, + * getting it handed off as wrapped transports, and then the writing of + * reponse data back to the client. In the process it manages flipping the + * read and write bits on the selection key for its client. + */ + protected class FrameBuffer { + // + // Possible states for the FrameBuffer state machine. + // + // in the midst of reading the frame size off the wire + private static final int READING_FRAME_SIZE = 1; + // reading the actual frame data now, but not all the way done yet + private static final int READING_FRAME = 2; + // completely read the frame, so an invocation can now happen + private static final int READ_FRAME_COMPLETE = 3; + // waiting to get switched to listening for write events + private static final int AWAITING_REGISTER_WRITE = 4; + // started writing response data, not fully complete yet + private static final int WRITING = 6; + // another thread wants this framebuffer to go back to reading + private static final int AWAITING_REGISTER_READ = 7; + // we want our transport and selection key invalidated in the selector thread + private static final int AWAITING_CLOSE = 8; + + // + // Instance variables + // + + // the actual transport hooked up to the client. + private final TNonblockingTransport trans_; + + // the SelectionKey that corresponds to our transport + private final SelectionKey selectionKey_; + + // where in the process of reading/writing are we? + private int state_ = READING_FRAME_SIZE; + + // the ByteBuffer we'll be using to write and read, depending on the state + private ByteBuffer buffer_; + + private TByteArrayOutputStream response_; + + public FrameBuffer( final TNonblockingTransport trans, + final SelectionKey selectionKey) { + trans_ = trans; + selectionKey_ = selectionKey; + buffer_ = ByteBuffer.allocate(4); + } + + /** + * Give this FrameBuffer a chance to read. The selector loop should have + * received a read event for this FrameBuffer. + * + * @return true if the connection should live on, false if it should be + * closed + */ + public boolean read() { + if (state_ == READING_FRAME_SIZE) { + // try to read the frame size completely + if (!internalRead()) { + return false; + } + + // if the frame size has been read completely, then prepare to read the + // actual frame. + if (buffer_.remaining() == 0) { + // pull out the frame size as an integer. + int frameSize = buffer_.getInt(0); + if (frameSize <= 0) { + LOGGER.error("Read an invalid frame size of " + frameSize + + ". Are you using TFramedTransport on the client side?"); + return false; + } + + // if this frame will always be too large for this server, log the + // error and close the connection. + if (frameSize + 4 > MAX_READ_BUFFER_BYTES) { + LOGGER.error("Read a frame size of " + frameSize + + ", which is bigger than the maximum allowable buffer size for ALL connections."); + return false; + } + + // if this frame will push us over the memory limit, then return. + // with luck, more memory will free up the next time around. + if (readBufferBytesAllocated + frameSize + 4 > MAX_READ_BUFFER_BYTES) { + return true; + } + + // incremement the amount of memory allocated to read buffers + readBufferBytesAllocated += frameSize + 4; + + // reallocate the readbuffer as a frame-sized buffer + buffer_ = ByteBuffer.allocate(frameSize + 4); + // put the frame size at the head of the buffer + buffer_.putInt(frameSize); + + state_ = READING_FRAME; + } else { + // this skips the check of READING_FRAME state below, since we can't + // possibly go on to that state if there's data left to be read at + // this one. + return true; + } + } + + // it is possible to fall through from the READING_FRAME_SIZE section + // to READING_FRAME if there's already some frame data available once + // READING_FRAME_SIZE is complete. + + if (state_ == READING_FRAME) { + if (!internalRead()) { + return false; + } + + // since we're already in the select loop here for sure, we can just + // modify our selection key directly. + if (buffer_.remaining() == 0) { + // get rid of the read select interests + selectionKey_.interestOps(0); + state_ = READ_FRAME_COMPLETE; + } + + return true; + } + + // if we fall through to this point, then the state must be invalid. + LOGGER.error("Read was called but state is invalid (" + state_ + ")"); + return false; + } + + /** + * Give this FrameBuffer a chance to write its output to the final client. + */ + public boolean write() { + if (state_ == WRITING) { + try { + if (trans_.write(buffer_) < 0) { + return false; + } + } catch (IOException e) { + LOGGER.warn("Got an IOException during write!", e); + return false; + } + + // we're done writing. now we need to switch back to reading. + if (buffer_.remaining() == 0) { + prepareRead(); + } + return true; + } + + LOGGER.error("Write was called, but state is invalid (" + state_ + ")"); + return false; + } + + /** + * Give this FrameBuffer a chance to set its interest to write, once data + * has come in. + */ + public void changeSelectInterests() { + if (state_ == AWAITING_REGISTER_WRITE) { + // set the OP_WRITE interest + selectionKey_.interestOps(SelectionKey.OP_WRITE); + state_ = WRITING; + } else if (state_ == AWAITING_REGISTER_READ) { + prepareRead(); + } else if (state_ == AWAITING_CLOSE){ + close(); + selectionKey_.cancel(); + } else { + LOGGER.error( + "changeSelectInterest was called, but state is invalid (" + + state_ + ")"); + } + } + + /** + * Shut the connection down. + */ + public void close() { + // if we're being closed due to an error, we might have allocated a + // buffer that we need to subtract for our memory accounting. + if (state_ == READING_FRAME || state_ == READ_FRAME_COMPLETE) { + readBufferBytesAllocated -= buffer_.array().length; + } + trans_.close(); + } + + /** + * Check if this FrameBuffer has a full frame read. + */ + public boolean isFrameFullyRead() { + return state_ == READ_FRAME_COMPLETE; + } + + /** + * After the processor has processed the invocation, whatever thread is + * managing invocations should call this method on this FrameBuffer so we + * know it's time to start trying to write again. Also, if it turns out + * that there actually isn't any data in the response buffer, we'll skip + * trying to write and instead go back to reading. + */ + public void responseReady() { + // the read buffer is definitely no longer in use, so we will decrement + // our read buffer count. we do this here as well as in close because + // we'd like to free this read memory up as quickly as possible for other + // clients. + readBufferBytesAllocated -= buffer_.array().length; + + if (response_.len() == 0) { + // go straight to reading again. this was probably an oneway method + state_ = AWAITING_REGISTER_READ; + buffer_ = null; + } else { + buffer_ = ByteBuffer.wrap(response_.get(), 0, response_.len()); + + // set state that we're waiting to be switched to write. we do this + // asynchronously through requestSelectInterestChange() because there is a + // possibility that we're not in the main thread, and thus currently + // blocked in select(). (this functionality is in place for the sake of + // the HsHa server.) + state_ = AWAITING_REGISTER_WRITE; + } + requestSelectInterestChange(); + } + + /** + * Actually invoke the method signified by this FrameBuffer. + */ + public void invoke() { + TTransport inTrans = getInputTransport(); + TProtocol inProt = inputProtocolFactory_.getProtocol(inTrans); + TProtocol outProt = outputProtocolFactory_.getProtocol(getOutputTransport()); + + try { + processorFactory_.getProcessor(inTrans).process(inProt, outProt); + responseReady(); + return; + } catch (TException te) { + LOGGER.warn("Exception while invoking!", te); + } catch (Exception e) { + LOGGER.error("Unexpected exception while invoking!", e); + } + // This will only be reached when there is an exception. + state_ = AWAITING_CLOSE; + requestSelectInterestChange(); + } + + /** + * Wrap the read buffer in a memory-based transport so a processor can read + * the data it needs to handle an invocation. + */ + private TTransport getInputTransport() { + return inputTransportFactory_.getTransport(new TIOStreamTransport( + new ByteArrayInputStream(buffer_.array()))); + } + + /** + * Get the transport that should be used by the invoker for responding. + */ + private TTransport getOutputTransport() { + response_ = new TByteArrayOutputStream(); + return outputTransportFactory_.getTransport(new TIOStreamTransport(response_)); + } + + /** + * Perform a read into buffer. + * + * @return true if the read succeeded, false if there was an error or the + * connection closed. + */ + private boolean internalRead() { + try { + if (trans_.read(buffer_) < 0) { + return false; + } + return true; + } catch (IOException e) { + LOGGER.warn("Got an IOException in internalRead!", e); + return false; + } + } + + /** + * We're done writing, so reset our interest ops and change state accordingly. + */ + private void prepareRead() { + // we can set our interest directly without using the queue because + // we're in the select thread. + selectionKey_.interestOps(SelectionKey.OP_READ); + // get ready for another go-around + buffer_ = ByteBuffer.allocate(4); + state_ = READING_FRAME_SIZE; + } + + /** + * When this FrameBuffer needs to change it's select interests and execution + * might not be in the select thread, then this method will make sure the + * interest change gets done when the select thread wakes back up. When the + * current thread is the select thread, then it just does the interest change + * immediately. + */ + private void requestSelectInterestChange() { + if (Thread.currentThread() == selectThread_) { + changeSelectInterests(); + } else { + TNonblockingServer.this.requestSelectInterestChange(this); + } + } + } // FrameBuffer + + + public static class Options { + public long maxReadBufferBytes = Long.MAX_VALUE; + + public Options() {} + + public void validate() { + if (maxReadBufferBytes <= 1024) { + throw new IllegalArgumentException("You must allocate at least 1KB to the read buffer."); + } + } + } +} diff --git a/lib/java/src/org/apache/thrift/server/TServer.java b/lib/java/src/org/apache/thrift/server/TServer.java new file mode 100644 index 00000000..eafe0c17 --- /dev/null +++ b/lib/java/src/org/apache/thrift/server/TServer.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import org.apache.thrift.TProcessorFactory; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TServerTransport; +import org.apache.thrift.transport.TTransportFactory; + +/** + * Generic interface for a Thrift server. + * + */ +public abstract class TServer { + + /** + * Core processor + */ + protected TProcessorFactory processorFactory_; + + /** + * Server transport + */ + protected TServerTransport serverTransport_; + + /** + * Input Transport Factory + */ + protected TTransportFactory inputTransportFactory_; + + /** + * Output Transport Factory + */ + protected TTransportFactory outputTransportFactory_; + + /** + * Input Protocol Factory + */ + protected TProtocolFactory inputProtocolFactory_; + + /** + * Output Protocol Factory + */ + protected TProtocolFactory outputProtocolFactory_; + + /** + * Default constructors. + */ + + protected TServer(TProcessorFactory processorFactory, + TServerTransport serverTransport) { + this(processorFactory, + serverTransport, + new TTransportFactory(), + new TTransportFactory(), + new TBinaryProtocol.Factory(), + new TBinaryProtocol.Factory()); + } + + protected TServer(TProcessorFactory processorFactory, + TServerTransport serverTransport, + TTransportFactory transportFactory) { + this(processorFactory, + serverTransport, + transportFactory, + transportFactory, + new TBinaryProtocol.Factory(), + new TBinaryProtocol.Factory()); + } + + protected TServer(TProcessorFactory processorFactory, + TServerTransport serverTransport, + TTransportFactory transportFactory, + TProtocolFactory protocolFactory) { + this(processorFactory, + serverTransport, + transportFactory, + transportFactory, + protocolFactory, + protocolFactory); + } + + protected TServer(TProcessorFactory processorFactory, + TServerTransport serverTransport, + TTransportFactory inputTransportFactory, + TTransportFactory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory) { + processorFactory_ = processorFactory; + serverTransport_ = serverTransport; + inputTransportFactory_ = inputTransportFactory; + outputTransportFactory_ = outputTransportFactory; + inputProtocolFactory_ = inputProtocolFactory; + outputProtocolFactory_ = outputProtocolFactory; + } + + /** + * The run method fires up the server and gets things going. + */ + public abstract void serve(); + + /** + * Stop the server. This is optional on a per-implementation basis. Not + * all servers are required to be cleanly stoppable. + */ + public void stop() {} + +} diff --git a/lib/java/src/org/apache/thrift/server/TSimpleServer.java b/lib/java/src/org/apache/thrift/server/TSimpleServer.java new file mode 100644 index 00000000..b3ee5ad6 --- /dev/null +++ b/lib/java/src/org/apache/thrift/server/TSimpleServer.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; +import org.apache.thrift.TProcessorFactory; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TServerTransport; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportFactory; +import org.apache.thrift.transport.TTransportException; +import org.apache.log4j.Logger; + +/** + * Simple singlethreaded server for testing. + * + */ +public class TSimpleServer extends TServer { + + private static final Logger LOGGER = Logger.getLogger(TSimpleServer.class.getName()); + + private boolean stopped_ = false; + + public TSimpleServer(TProcessor processor, + TServerTransport serverTransport) { + super(new TProcessorFactory(processor), serverTransport); + } + + public TSimpleServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory transportFactory, + TProtocolFactory protocolFactory) { + super(new TProcessorFactory(processor), serverTransport, transportFactory, protocolFactory); + } + + public TSimpleServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory inputTransportFactory, + TTransportFactory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory) { + super(new TProcessorFactory(processor), serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory); + } + + public TSimpleServer(TProcessorFactory processorFactory, + TServerTransport serverTransport) { + super(processorFactory, serverTransport); + } + + public TSimpleServer(TProcessorFactory processorFactory, + TServerTransport serverTransport, + TTransportFactory transportFactory, + TProtocolFactory protocolFactory) { + super(processorFactory, serverTransport, transportFactory, protocolFactory); + } + + public TSimpleServer(TProcessorFactory processorFactory, + TServerTransport serverTransport, + TTransportFactory inputTransportFactory, + TTransportFactory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory) { + super(processorFactory, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory); + } + + + public void serve() { + stopped_ = false; + try { + serverTransport_.listen(); + } catch (TTransportException ttx) { + LOGGER.error("Error occurred during listening.", ttx); + return; + } + + while (!stopped_) { + TTransport client = null; + TProcessor processor = null; + TTransport inputTransport = null; + TTransport outputTransport = null; + TProtocol inputProtocol = null; + TProtocol outputProtocol = null; + try { + client = serverTransport_.accept(); + if (client != null) { + processor = processorFactory_.getProcessor(client); + inputTransport = inputTransportFactory_.getTransport(client); + outputTransport = outputTransportFactory_.getTransport(client); + inputProtocol = inputProtocolFactory_.getProtocol(inputTransport); + outputProtocol = outputProtocolFactory_.getProtocol(outputTransport); + while (processor.process(inputProtocol, outputProtocol)) {} + } + } catch (TTransportException ttx) { + // Client died, just move on + } catch (TException tx) { + if (!stopped_) { + LOGGER.error("Thrift error occurred during processing of message.", tx); + } + } catch (Exception x) { + if (!stopped_) { + LOGGER.error("Error occurred during processing of message.", x); + } + } + + if (inputTransport != null) { + inputTransport.close(); + } + + if (outputTransport != null) { + outputTransport.close(); + } + + } + } + + public void stop() { + stopped_ = true; + serverTransport_.interrupt(); + } +} diff --git a/lib/java/src/org/apache/thrift/server/TThreadPoolServer.java b/lib/java/src/org/apache/thrift/server/TThreadPoolServer.java new file mode 100644 index 00000000..ebc5a9be --- /dev/null +++ b/lib/java/src/org/apache/thrift/server/TThreadPoolServer.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; +import org.apache.thrift.TProcessorFactory; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.transport.TServerTransport; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.transport.TTransportFactory; +import org.apache.log4j.Logger; +import org.apache.log4j.Level; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + + +/** + * Server which uses Java's built in ThreadPool management to spawn off + * a worker pool that + * + */ +public class TThreadPoolServer extends TServer { + + private static final Logger LOGGER = Logger.getLogger(TThreadPoolServer.class.getName()); + + // Executor service for handling client connections + private ExecutorService executorService_; + + // Flag for stopping the server + private volatile boolean stopped_; + + // Server options + private Options options_; + + // Customizable server options + public static class Options { + public int minWorkerThreads = 5; + public int maxWorkerThreads = Integer.MAX_VALUE; + public int stopTimeoutVal = 60; + public TimeUnit stopTimeoutUnit = TimeUnit.SECONDS; + } + + public TThreadPoolServer(TProcessor processor, + TServerTransport serverTransport) { + this(processor, serverTransport, + new TTransportFactory(), new TTransportFactory(), + new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory()); + } + + public TThreadPoolServer(TProcessorFactory processorFactory, + TServerTransport serverTransport) { + this(processorFactory, serverTransport, + new TTransportFactory(), new TTransportFactory(), + new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory()); + } + + public TThreadPoolServer(TProcessor processor, + TServerTransport serverTransport, + TProtocolFactory protocolFactory) { + this(processor, serverTransport, + new TTransportFactory(), new TTransportFactory(), + protocolFactory, protocolFactory); + } + + public TThreadPoolServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory transportFactory, + TProtocolFactory protocolFactory) { + this(processor, serverTransport, + transportFactory, transportFactory, + protocolFactory, protocolFactory); + } + + public TThreadPoolServer(TProcessorFactory processorFactory, + TServerTransport serverTransport, + TTransportFactory transportFactory, + TProtocolFactory protocolFactory) { + this(processorFactory, serverTransport, + transportFactory, transportFactory, + protocolFactory, protocolFactory); + } + + public TThreadPoolServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory inputTransportFactory, + TTransportFactory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory) { + this(new TProcessorFactory(processor), serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory); + } + + public TThreadPoolServer(TProcessorFactory processorFactory, + TServerTransport serverTransport, + TTransportFactory inputTransportFactory, + TTransportFactory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory) { + super(processorFactory, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory); + options_ = new Options(); + executorService_ = Executors.newCachedThreadPool(); + } + + public TThreadPoolServer(TProcessor processor, + TServerTransport serverTransport, + TTransportFactory inputTransportFactory, + TTransportFactory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory, + Options options) { + this(new TProcessorFactory(processor), serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory, + options); + } + + public TThreadPoolServer(TProcessorFactory processorFactory, + TServerTransport serverTransport, + TTransportFactory inputTransportFactory, + TTransportFactory outputTransportFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory, + Options options) { + super(processorFactory, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory); + + executorService_ = null; + + SynchronousQueue executorQueue = + new SynchronousQueue(); + + executorService_ = new ThreadPoolExecutor(options.minWorkerThreads, + options.maxWorkerThreads, + 60, + TimeUnit.SECONDS, + executorQueue); + + options_ = options; + } + + + public void serve() { + try { + serverTransport_.listen(); + } catch (TTransportException ttx) { + LOGGER.error("Error occurred during listening.", ttx); + return; + } + + stopped_ = false; + while (!stopped_) { + int failureCount = 0; + try { + TTransport client = serverTransport_.accept(); + WorkerProcess wp = new WorkerProcess(client); + executorService_.execute(wp); + } catch (TTransportException ttx) { + if (!stopped_) { + ++failureCount; + LOGGER.warn("Transport error occurred during acceptance of message.", ttx); + } + } + } + + executorService_.shutdown(); + + // Loop until awaitTermination finally does return without a interrupted + // exception. If we don't do this, then we'll shut down prematurely. We want + // to let the executorService clear it's task queue, closing client sockets + // appropriately. + long timeoutMS = options_.stopTimeoutUnit.toMillis(options_.stopTimeoutVal); + long now = System.currentTimeMillis(); + while (timeoutMS >= 0) { + try { + executorService_.awaitTermination(timeoutMS, TimeUnit.MILLISECONDS); + break; + } catch (InterruptedException ix) { + long newnow = System.currentTimeMillis(); + timeoutMS -= (newnow - now); + now = newnow; + } + } + } + + public void stop() { + stopped_ = true; + serverTransport_.interrupt(); + } + + private class WorkerProcess implements Runnable { + + /** + * Client that this services. + */ + private TTransport client_; + + /** + * Default constructor. + * + * @param client Transport to process + */ + private WorkerProcess(TTransport client) { + client_ = client; + } + + /** + * Loops on processing a client forever + */ + public void run() { + TProcessor processor = null; + TTransport inputTransport = null; + TTransport outputTransport = null; + TProtocol inputProtocol = null; + TProtocol outputProtocol = null; + try { + processor = processorFactory_.getProcessor(client_); + inputTransport = inputTransportFactory_.getTransport(client_); + outputTransport = outputTransportFactory_.getTransport(client_); + inputProtocol = inputProtocolFactory_.getProtocol(inputTransport); + outputProtocol = outputProtocolFactory_.getProtocol(outputTransport); + // we check stopped_ first to make sure we're not supposed to be shutting + // down. this is necessary for graceful shutdown. + while (!stopped_ && processor.process(inputProtocol, outputProtocol)) {} + } catch (TTransportException ttx) { + // Assume the client died and continue silently + } catch (TException tx) { + LOGGER.error("Thrift error occurred during processing of message.", tx); + } catch (Exception x) { + LOGGER.error("Error occurred during processing of message.", x); + } + + if (inputTransport != null) { + inputTransport.close(); + } + + if (outputTransport != null) { + outputTransport.close(); + } + } + } +} diff --git a/lib/java/src/org/apache/thrift/transport/TFramedTransport.java b/lib/java/src/org/apache/thrift/transport/TFramedTransport.java new file mode 100644 index 00000000..c83748ad --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TFramedTransport.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.ByteArrayInputStream; + +import org.apache.thrift.TByteArrayOutputStream; + +/** + * Socket implementation of the TTransport interface. To be commented soon! + * + */ +public class TFramedTransport extends TTransport { + + /** + * Underlying transport + */ + private TTransport transport_ = null; + + /** + * Buffer for output + */ + private final TByteArrayOutputStream writeBuffer_ = + new TByteArrayOutputStream(1024); + + /** + * Buffer for input + */ + private ByteArrayInputStream readBuffer_ = null; + + public static class Factory extends TTransportFactory { + public Factory() { + } + + public TTransport getTransport(TTransport base) { + return new TFramedTransport(base); + } + } + + /** + * Constructor wraps around another tranpsort + */ + public TFramedTransport(TTransport transport) { + transport_ = transport; + } + + public void open() throws TTransportException { + transport_.open(); + } + + public boolean isOpen() { + return transport_.isOpen(); + } + + public void close() { + transport_.close(); + } + + public int read(byte[] buf, int off, int len) throws TTransportException { + if (readBuffer_ != null) { + int got = readBuffer_.read(buf, off, len); + if (got > 0) { + return got; + } + } + + // Read another frame of data + readFrame(); + + return readBuffer_.read(buf, off, len); + } + + private void readFrame() throws TTransportException { + byte[] i32rd = new byte[4]; + transport_.readAll(i32rd, 0, 4); + int size = + ((i32rd[0] & 0xff) << 24) | + ((i32rd[1] & 0xff) << 16) | + ((i32rd[2] & 0xff) << 8) | + ((i32rd[3] & 0xff)); + + byte[] buff = new byte[size]; + transport_.readAll(buff, 0, size); + readBuffer_ = new ByteArrayInputStream(buff); + } + + public void write(byte[] buf, int off, int len) throws TTransportException { + writeBuffer_.write(buf, off, len); + } + + public void flush() throws TTransportException { + byte[] buf = writeBuffer_.get(); + int len = writeBuffer_.len(); + writeBuffer_.reset(); + + byte[] i32out = new byte[4]; + i32out[0] = (byte)(0xff & (len >> 24)); + i32out[1] = (byte)(0xff & (len >> 16)); + i32out[2] = (byte)(0xff & (len >> 8)); + i32out[3] = (byte)(0xff & (len)); + transport_.write(i32out, 0, 4); + transport_.write(buf, 0, len); + transport_.flush(); + } +} diff --git a/lib/java/src/org/apache/thrift/transport/THttpClient.java b/lib/java/src/org/apache/thrift/transport/THttpClient.java new file mode 100644 index 00000000..41923531 --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/THttpClient.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.IOException; + +import java.net.URL; +import java.net.HttpURLConnection; +import java.util.HashMap; +import java.util.Map; + +/** + * HTTP implementation of the TTransport interface. Used for working with a + * Thrift web services implementation. + * + */ +public class THttpClient extends TTransport { + + private URL url_ = null; + + private final ByteArrayOutputStream requestBuffer_ = + new ByteArrayOutputStream(); + + private InputStream inputStream_ = null; + + private int connectTimeout_ = 0; + + private int readTimeout_ = 0; + + private Map customHeaders_ = null; + + public THttpClient(String url) throws TTransportException { + try { + url_ = new URL(url); + } catch (IOException iox) { + throw new TTransportException(iox); + } + } + + public void setConnectTimeout(int timeout) { + connectTimeout_ = timeout; + } + + public void setReadTimeout(int timeout) { + readTimeout_ = timeout; + } + + public void setCustomHeaders(Map headers) { + customHeaders_ = headers; + } + + public void setCustomHeader(String key, String value) { + if (customHeaders_ == null) { + customHeaders_ = new HashMap(); + } + customHeaders_.put(key, value); + } + + public void open() {} + + public void close() { + if (null != inputStream_) { + try { + inputStream_.close(); + } catch (IOException ioe) { + ; + } + inputStream_ = null; + } + } + + public boolean isOpen() { + return true; + } + + public int read(byte[] buf, int off, int len) throws TTransportException { + if (inputStream_ == null) { + throw new TTransportException("Response buffer is empty, no request."); + } + try { + int ret = inputStream_.read(buf, off, len); + if (ret == -1) { + throw new TTransportException("No more data available."); + } + return ret; + } catch (IOException iox) { + throw new TTransportException(iox); + } + } + + public void write(byte[] buf, int off, int len) { + requestBuffer_.write(buf, off, len); + } + + public void flush() throws TTransportException { + // Extract request and reset buffer + byte[] data = requestBuffer_.toByteArray(); + requestBuffer_.reset(); + + try { + // Create connection object + HttpURLConnection connection = (HttpURLConnection)url_.openConnection(); + + // Timeouts, only if explicitly set + if (connectTimeout_ > 0) { + connection.setConnectTimeout(connectTimeout_); + } + if (readTimeout_ > 0) { + connection.setReadTimeout(readTimeout_); + } + + // Make the request + connection.setRequestMethod("POST"); + connection.setRequestProperty("Content-Type", "application/x-thrift"); + connection.setRequestProperty("Accept", "application/x-thrift"); + connection.setRequestProperty("User-Agent", "Java/THttpClient"); + if (customHeaders_ != null) { + for (Map.Entry header : customHeaders_.entrySet()) { + connection.setRequestProperty(header.getKey(), header.getValue()); + } + } + connection.setDoOutput(true); + connection.connect(); + connection.getOutputStream().write(data); + + int responseCode = connection.getResponseCode(); + if (responseCode != HttpURLConnection.HTTP_OK) { + throw new TTransportException("HTTP Response code: " + responseCode); + } + + // Read the responses + inputStream_ = connection.getInputStream(); + + } catch (IOException iox) { + throw new TTransportException(iox); + } + } +} diff --git a/lib/java/src/org/apache/thrift/transport/TIOStreamTransport.java b/lib/java/src/org/apache/thrift/transport/TIOStreamTransport.java new file mode 100644 index 00000000..89cdb582 --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TIOStreamTransport.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * This is the most commonly used base transport. It takes an InputStream + * and an OutputStream and uses those to perform all transport operations. + * This allows for compatibility with all the nice constructs Java already + * has to provide a variety of types of streams. + * + */ +public class TIOStreamTransport extends TTransport { + + private static final Logger LOGGER = Logger.getLogger(TIOStreamTransport.class.getName()); + + /** Underlying inputStream */ + protected InputStream inputStream_ = null; + + /** Underlying outputStream */ + protected OutputStream outputStream_ = null; + + /** + * Subclasses can invoke the default constructor and then assign the input + * streams in the open method. + */ + protected TIOStreamTransport() {} + + /** + * Input stream constructor. + * + * @param is Input stream to read from + */ + public TIOStreamTransport(InputStream is) { + inputStream_ = is; + } + + /** + * Output stream constructor. + * + * @param os Output stream to read from + */ + public TIOStreamTransport(OutputStream os) { + outputStream_ = os; + } + + /** + * Two-way stream constructor. + * + * @param is Input stream to read from + * @param os Output stream to read from + */ + public TIOStreamTransport(InputStream is, OutputStream os) { + inputStream_ = is; + outputStream_ = os; + } + + /** + * The streams must already be open at construction time, so this should + * always return true. + * + * @return true + */ + public boolean isOpen() { + return true; + } + + /** + * The streams must already be open. This method does nothing. + */ + public void open() throws TTransportException {} + + /** + * Closes both the input and output streams. + */ + public void close() { + if (inputStream_ != null) { + try { + inputStream_.close(); + } catch (IOException iox) { + LOGGER.warn("Error closing input stream.", iox); + } + inputStream_ = null; + } + if (outputStream_ != null) { + try { + outputStream_.close(); + } catch (IOException iox) { + LOGGER.warn("Error closing output stream.", iox); + } + outputStream_ = null; + } + } + + /** + * Reads from the underlying input stream if not null. + */ + public int read(byte[] buf, int off, int len) throws TTransportException { + if (inputStream_ == null) { + throw new TTransportException(TTransportException.NOT_OPEN, "Cannot read from null inputStream"); + } + try { + return inputStream_.read(buf, off, len); + } catch (IOException iox) { + throw new TTransportException(TTransportException.UNKNOWN, iox); + } + } + + /** + * Writes to the underlying output stream if not null. + */ + public void write(byte[] buf, int off, int len) throws TTransportException { + if (outputStream_ == null) { + throw new TTransportException(TTransportException.NOT_OPEN, "Cannot write to null outputStream"); + } + try { + outputStream_.write(buf, off, len); + } catch (IOException iox) { + throw new TTransportException(TTransportException.UNKNOWN, iox); + } + } + + /** + * Flushes the underlying output stream if not null. + */ + public void flush() throws TTransportException { + if (outputStream_ == null) { + throw new TTransportException(TTransportException.NOT_OPEN, "Cannot flush null outputStream"); + } + try { + outputStream_.flush(); + } catch (IOException iox) { + throw new TTransportException(TTransportException.UNKNOWN, iox); + } + } +} diff --git a/lib/java/src/org/apache/thrift/transport/TMemoryBuffer.java b/lib/java/src/org/apache/thrift/transport/TMemoryBuffer.java new file mode 100644 index 00000000..886fcbf6 --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TMemoryBuffer.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import org.apache.thrift.TByteArrayOutputStream; +import java.io.UnsupportedEncodingException; + +/** + * Memory buffer-based implementation of the TTransport interface. + * + */ +public class TMemoryBuffer extends TTransport { + + /** + * + */ + public TMemoryBuffer(int size) { + arr_ = new TByteArrayOutputStream(size); + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void open() { + /* Do nothing */ + } + + @Override + public void close() { + /* Do nothing */ + } + + @Override + public int read(byte[] buf, int off, int len) { + byte[] src = arr_.get(); + int amtToRead = (len > arr_.len() - pos_ ? arr_.len() - pos_ : len); + if (amtToRead > 0) { + System.arraycopy(src, pos_, buf, off, amtToRead); + pos_ += amtToRead; + } + return amtToRead; + } + + @Override + public void write(byte[] buf, int off, int len) { + arr_.write(buf, off, len); + } + + /** + * Output the contents of the memory buffer as a String, using the supplied + * encoding + * @param enc the encoding to use + * @return the contents of the memory buffer as a String + */ + public String toString(String enc) throws UnsupportedEncodingException { + return arr_.toString(enc); + } + + public String inspect() { + String buf = ""; + byte[] bytes = arr_.toByteArray(); + for (int i = 0; i < bytes.length; i++) { + buf += (pos_ == i ? "==>" : "" ) + Integer.toHexString(bytes[i] & 0xff) + " "; + } + return buf; + } + + // The contents of the buffer + private TByteArrayOutputStream arr_; + + // Position to read next byte from + private int pos_; + + public int length() { + return arr_.size(); + } +} + diff --git a/lib/java/src/org/apache/thrift/transport/TNonblockingServerSocket.java b/lib/java/src/org/apache/thrift/transport/TNonblockingServerSocket.java new file mode 100644 index 00000000..571adbff --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TNonblockingServerSocket.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.thrift.transport; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.SocketException; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; + +/** + * Wrapper around ServerSocketChannel + */ +public class TNonblockingServerSocket extends TNonblockingServerTransport { + + /** + * This channel is where all the nonblocking magic happens. + */ + private ServerSocketChannel serverSocketChannel = null; + + /** + * Underlying serversocket object + */ + private ServerSocket serverSocket_ = null; + + /** + * Port to listen on + */ + private int port_ = 0; + + /** + * Timeout for client sockets from accept + */ + private int clientTimeout_ = 0; + + /** + * Creates a server socket from underlying socket object + */ + // public TNonblockingServerSocket(ServerSocket serverSocket) { + // this(serverSocket, 0); + // } + + /** + * Creates a server socket from underlying socket object + */ + // public TNonblockingServerSocket(ServerSocket serverSocket, int clientTimeout) { + // serverSocket_ = serverSocket; + // clientTimeout_ = clientTimeout; + // } + + /** + * Creates just a port listening server socket + */ + public TNonblockingServerSocket(int port) throws TTransportException { + this(port, 0); + } + + /** + * Creates just a port listening server socket + */ + public TNonblockingServerSocket(int port, int clientTimeout) throws TTransportException { + port_ = port; + clientTimeout_ = clientTimeout; + try { + serverSocketChannel = ServerSocketChannel.open(); + serverSocketChannel.configureBlocking(false); + + // Make server socket + serverSocket_ = serverSocketChannel.socket(); + // Prevent 2MSL delay problem on server restarts + serverSocket_.setReuseAddress(true); + // Bind to listening port + serverSocket_.bind(new InetSocketAddress(port_)); + } catch (IOException ioe) { + serverSocket_ = null; + throw new TTransportException("Could not create ServerSocket on port " + port + "."); + } + } + + public void listen() throws TTransportException { + // Make sure not to block on accept + if (serverSocket_ != null) { + try { + serverSocket_.setSoTimeout(0); + } catch (SocketException sx) { + sx.printStackTrace(); + } + } + } + + protected TNonblockingSocket acceptImpl() throws TTransportException { + if (serverSocket_ == null) { + throw new TTransportException(TTransportException.NOT_OPEN, "No underlying server socket."); + } + try { + SocketChannel socketChannel = serverSocketChannel.accept(); + if (socketChannel == null) { + return null; + } + + TNonblockingSocket tsocket = new TNonblockingSocket(socketChannel); + tsocket.setTimeout(clientTimeout_); + return tsocket; + } catch (IOException iox) { + throw new TTransportException(iox); + } + } + + public void registerSelector(Selector selector) { + try { + // Register the server socket channel, indicating an interest in + // accepting new connections + serverSocketChannel.register(selector, SelectionKey.OP_ACCEPT); + } catch (ClosedChannelException e) { + // this shouldn't happen, ideally... + // TODO: decide what to do with this. + } + } + + public void close() { + if (serverSocket_ != null) { + try { + serverSocket_.close(); + } catch (IOException iox) { + System.err.println("WARNING: Could not close server socket: " + + iox.getMessage()); + } + serverSocket_ = null; + } + } + + public void interrupt() { + // The thread-safeness of this is dubious, but Java documentation suggests + // that it is safe to do this from a different thread context + close(); + } + +} diff --git a/lib/java/src/org/apache/thrift/transport/TNonblockingServerTransport.java b/lib/java/src/org/apache/thrift/transport/TNonblockingServerTransport.java new file mode 100644 index 00000000..ba45b09d --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TNonblockingServerTransport.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.thrift.transport; + +import java.nio.channels.Selector; + +/** + * Server transport that can be operated in a nonblocking fashion. + */ +public abstract class TNonblockingServerTransport extends TServerTransport { + + public abstract void registerSelector(Selector selector); +} diff --git a/lib/java/src/org/apache/thrift/transport/TNonblockingSocket.java b/lib/java/src/org/apache/thrift/transport/TNonblockingSocket.java new file mode 100644 index 00000000..bc2d5396 --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TNonblockingSocket.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.thrift.transport; + +import java.io.IOException; +import java.net.Socket; +import java.net.SocketException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; + +/** + * Socket implementation of the TTransport interface. To be commented soon! + */ +public class TNonblockingSocket extends TNonblockingTransport { + + private SocketChannel socketChannel = null; + + /** + * Wrapped Socket object + */ + private Socket socket_ = null; + + /** + * Remote host + */ + private String host_ = null; + + /** + * Remote port + */ + private int port_ = 0; + + /** + * Socket timeout + */ + private int timeout_ = 0; + + /** + * Constructor that takes an already created socket. + * + * @param socketChannel Already created SocketChannel object + * @throws TTransportException if there is an error setting up the streams + */ + public TNonblockingSocket(SocketChannel socketChannel) throws TTransportException { + try { + // make it a nonblocking channel + socketChannel.configureBlocking(false); + } catch (IOException e) { + throw new TTransportException(e); + } + + this.socketChannel = socketChannel; + this.socket_ = socketChannel.socket(); + try { + socket_.setSoLinger(false, 0); + socket_.setTcpNoDelay(true); + } catch (SocketException sx) { + sx.printStackTrace(); + } + } + + /** + * Register this socket with the specified selector for both read and write + * operations. + * + * @param selector + * @return the selection key for this socket. + */ + public SelectionKey registerSelector(Selector selector, int interests) throws IOException { + // Register the new SocketChannel with our Selector, indicating + // we'd like to be notified when there's data waiting to be read + return socketChannel.register(selector, interests); + } + + /** + * Initializes the socket object + */ + private void initSocket() { + socket_ = new Socket(); + try { + socket_.setSoLinger(false, 0); + socket_.setTcpNoDelay(true); + socket_.setSoTimeout(timeout_); + } catch (SocketException sx) { + sx.printStackTrace(); + } + } + + /** + * Sets the socket timeout + * + * @param timeout Milliseconds timeout + */ + public void setTimeout(int timeout) { + timeout_ = timeout; + try { + socket_.setSoTimeout(timeout); + } catch (SocketException sx) { + sx.printStackTrace(); + } + } + + /** + * Returns a reference to the underlying socket. + */ + public Socket getSocket() { + if (socket_ == null) { + initSocket(); + } + return socket_; + } + + /** + * Checks whether the socket is connected. + */ + public boolean isOpen() { + if (socket_ == null) { + return false; + } + return socket_.isConnected(); + } + + /** + * Connects the socket, creating a new socket object if necessary. + */ + public void open() throws TTransportException { + throw new RuntimeException("Not implemented yet"); + } + + /** + * Perform a nonblocking read into buffer. + */ + public int read(ByteBuffer buffer) throws IOException { + return socketChannel.read(buffer); + } + + + /** + * Reads from the underlying input stream if not null. + */ + public int read(byte[] buf, int off, int len) throws TTransportException { + if ((socketChannel.validOps() & SelectionKey.OP_READ) != SelectionKey.OP_READ) { + throw new TTransportException(TTransportException.NOT_OPEN, + "Cannot read from write-only socket channel"); + } + try { + return socketChannel.read(ByteBuffer.wrap(buf, off, len)); + } catch (IOException iox) { + throw new TTransportException(TTransportException.UNKNOWN, iox); + } + } + + /** + * Perform a nonblocking write of the data in buffer; + */ + public int write(ByteBuffer buffer) throws IOException { + return socketChannel.write(buffer); + } + + /** + * Writes to the underlying output stream if not null. + */ + public void write(byte[] buf, int off, int len) throws TTransportException { + if ((socketChannel.validOps() & SelectionKey.OP_WRITE) != SelectionKey.OP_WRITE) { + throw new TTransportException(TTransportException.NOT_OPEN, + "Cannot write to write-only socket channel"); + } + try { + socketChannel.write(ByteBuffer.wrap(buf, off, len)); + } catch (IOException iox) { + throw new TTransportException(TTransportException.UNKNOWN, iox); + } + } + + /** + * Flushes the underlying output stream if not null. + */ + public void flush() throws TTransportException { + // Not supported by SocketChannel. + } + + /** + * Closes the socket. + */ + public void close() { + try { + socketChannel.close(); + } catch (IOException e) { + // silently ignore. + } + } + +} diff --git a/lib/java/src/org/apache/thrift/transport/TNonblockingTransport.java b/lib/java/src/org/apache/thrift/transport/TNonblockingTransport.java new file mode 100644 index 00000000..517eacb7 --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TNonblockingTransport.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.IOException; +import java.nio.channels.Selector; +import java.nio.channels.SelectionKey; +import java.nio.ByteBuffer; + +public abstract class TNonblockingTransport extends TTransport { + public abstract SelectionKey registerSelector(Selector selector, int interests) throws IOException; + public abstract int read(ByteBuffer buffer) throws IOException; + public abstract int write(ByteBuffer buffer) throws IOException; +} diff --git a/lib/java/src/org/apache/thrift/transport/TServerSocket.java b/lib/java/src/org/apache/thrift/transport/TServerSocket.java new file mode 100644 index 00000000..796cd659 --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TServerSocket.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import org.apache.log4j.Logger; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketException; + +/** + * Wrapper around ServerSocket for Thrift. + * + */ +public class TServerSocket extends TServerTransport { + + private static final Logger LOGGER = Logger.getLogger(TServerSocket.class.getName()); + + /** + * Underlying serversocket object + */ + private ServerSocket serverSocket_ = null; + + /** + * Port to listen on + */ + private int port_ = 0; + + /** + * Timeout for client sockets from accept + */ + private int clientTimeout_ = 0; + + /** + * Creates a server socket from underlying socket object + */ + public TServerSocket(ServerSocket serverSocket) { + this(serverSocket, 0); + } + + /** + * Creates a server socket from underlying socket object + */ + public TServerSocket(ServerSocket serverSocket, int clientTimeout) { + serverSocket_ = serverSocket; + clientTimeout_ = clientTimeout; + } + + /** + * Creates just a port listening server socket + */ + public TServerSocket(int port) throws TTransportException { + this(port, 0); + } + + /** + * Creates just a port listening server socket + */ + public TServerSocket(int port, int clientTimeout) throws TTransportException { + this(new InetSocketAddress(port), clientTimeout); + port_ = port; + } + + public TServerSocket(InetSocketAddress bindAddr) throws TTransportException { + this(bindAddr, 0); + } + + public TServerSocket(InetSocketAddress bindAddr, int clientTimeout) throws TTransportException { + clientTimeout_ = clientTimeout; + try { + // Make server socket + serverSocket_ = new ServerSocket(); + // Prevent 2MSL delay problem on server restarts + serverSocket_.setReuseAddress(true); + // Bind to listening port + serverSocket_.bind(bindAddr); + } catch (IOException ioe) { + serverSocket_ = null; + throw new TTransportException("Could not create ServerSocket on address " + bindAddr.toString() + "."); + } + } + + public void listen() throws TTransportException { + // Make sure not to block on accept + if (serverSocket_ != null) { + try { + serverSocket_.setSoTimeout(0); + } catch (SocketException sx) { + LOGGER.error("Could not set socket timeout.", sx); + } + } + } + + protected TSocket acceptImpl() throws TTransportException { + if (serverSocket_ == null) { + throw new TTransportException(TTransportException.NOT_OPEN, "No underlying server socket."); + } + try { + Socket result = serverSocket_.accept(); + TSocket result2 = new TSocket(result); + result2.setTimeout(clientTimeout_); + return result2; + } catch (IOException iox) { + throw new TTransportException(iox); + } + } + + public void close() { + if (serverSocket_ != null) { + try { + serverSocket_.close(); + } catch (IOException iox) { + LOGGER.warn("Could not close server socket.", iox); + } + serverSocket_ = null; + } + } + + public void interrupt() { + // The thread-safeness of this is dubious, but Java documentation suggests + // that it is safe to do this from a different thread context + close(); + } + +} diff --git a/lib/java/src/org/apache/thrift/transport/TServerTransport.java b/lib/java/src/org/apache/thrift/transport/TServerTransport.java new file mode 100644 index 00000000..17ff86be --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TServerTransport.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +/** + * Server transport. Object which provides client transports. + * + */ +public abstract class TServerTransport { + + public abstract void listen() throws TTransportException; + + public final TTransport accept() throws TTransportException { + TTransport transport = acceptImpl(); + if (transport == null) { + throw new TTransportException("accept() may not return NULL"); + } + return transport; + } + + public abstract void close(); + + protected abstract TTransport acceptImpl() throws TTransportException; + + /** + * Optional method implementation. This signals to the server transport + * that it should break out of any accept() or listen() that it is currently + * blocked on. This method, if implemented, MUST be thread safe, as it may + * be called from a different thread context than the other TServerTransport + * methods. + */ + public void interrupt() {} + +} diff --git a/lib/java/src/org/apache/thrift/transport/TSocket.java b/lib/java/src/org/apache/thrift/transport/TSocket.java new file mode 100644 index 00000000..cdf1bcc4 --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TSocket.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import org.apache.log4j.Logger; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketException; + +/** + * Socket implementation of the TTransport interface. To be commented soon! + * + */ +public class TSocket extends TIOStreamTransport { + + private static final Logger LOGGER = Logger.getLogger(TSocket.class.getName()); + + /** + * Wrapped Socket object + */ + private Socket socket_ = null; + + /** + * Remote host + */ + private String host_ = null; + + /** + * Remote port + */ + private int port_ = 0; + + /** + * Socket timeout + */ + private int timeout_ = 0; + + /** + * Constructor that takes an already created socket. + * + * @param socket Already created socket object + * @throws TTransportException if there is an error setting up the streams + */ + public TSocket(Socket socket) throws TTransportException { + socket_ = socket; + try { + socket_.setSoLinger(false, 0); + socket_.setTcpNoDelay(true); + } catch (SocketException sx) { + LOGGER.warn("Could not configure socket.", sx); + } + + if (isOpen()) { + try { + inputStream_ = new BufferedInputStream(socket_.getInputStream(), 1024); + outputStream_ = new BufferedOutputStream(socket_.getOutputStream(), 1024); + } catch (IOException iox) { + close(); + throw new TTransportException(TTransportException.NOT_OPEN, iox); + } + } + } + + /** + * Creates a new unconnected socket that will connect to the given host + * on the given port. + * + * @param host Remote host + * @param port Remote port + */ + public TSocket(String host, int port) { + this(host, port, 0); + } + + /** + * Creates a new unconnected socket that will connect to the given host + * on the given port. + * + * @param host Remote host + * @param port Remote port + * @param timeout Socket timeout + */ + public TSocket(String host, int port, int timeout) { + host_ = host; + port_ = port; + timeout_ = timeout; + initSocket(); + } + + /** + * Initializes the socket object + */ + private void initSocket() { + socket_ = new Socket(); + try { + socket_.setSoLinger(false, 0); + socket_.setTcpNoDelay(true); + socket_.setSoTimeout(timeout_); + } catch (SocketException sx) { + LOGGER.error("Could not configure socket.", sx); + } + } + + /** + * Sets the socket timeout + * + * @param timeout Milliseconds timeout + */ + public void setTimeout(int timeout) { + timeout_ = timeout; + try { + socket_.setSoTimeout(timeout); + } catch (SocketException sx) { + LOGGER.warn("Could not set socket timeout.", sx); + } + } + + /** + * Returns a reference to the underlying socket. + */ + public Socket getSocket() { + if (socket_ == null) { + initSocket(); + } + return socket_; + } + + /** + * Checks whether the socket is connected. + */ + public boolean isOpen() { + if (socket_ == null) { + return false; + } + return socket_.isConnected(); + } + + /** + * Connects the socket, creating a new socket object if necessary. + */ + public void open() throws TTransportException { + if (isOpen()) { + throw new TTransportException(TTransportException.ALREADY_OPEN, "Socket already connected."); + } + + if (host_.length() == 0) { + throw new TTransportException(TTransportException.NOT_OPEN, "Cannot open null host."); + } + if (port_ <= 0) { + throw new TTransportException(TTransportException.NOT_OPEN, "Cannot open without port."); + } + + if (socket_ == null) { + initSocket(); + } + + try { + socket_.connect(new InetSocketAddress(host_, port_)); + inputStream_ = new BufferedInputStream(socket_.getInputStream(), 1024); + outputStream_ = new BufferedOutputStream(socket_.getOutputStream(), 1024); + } catch (IOException iox) { + close(); + throw new TTransportException(TTransportException.NOT_OPEN, iox); + } + } + + /** + * Closes the socket. + */ + public void close() { + // Close the underlying streams + super.close(); + + // Close the socket + if (socket_ != null) { + try { + socket_.close(); + } catch (IOException iox) { + LOGGER.warn("Could not close socket.", iox); + } + socket_ = null; + } + } + +} diff --git a/lib/java/src/org/apache/thrift/transport/TTransport.java b/lib/java/src/org/apache/thrift/transport/TTransport.java new file mode 100644 index 00000000..a6c047bb --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TTransport.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +/** + * Generic class that encapsulates the I/O layer. This is basically a thin + * wrapper around the combined functionality of Java input/output streams. + * + */ +public abstract class TTransport { + + /** + * Queries whether the transport is open. + * + * @return True if the transport is open. + */ + public abstract boolean isOpen(); + + /** + * Is there more data to be read? + * + * @return True if the remote side is still alive and feeding us + */ + public boolean peek() { + return isOpen(); + } + + /** + * Opens the transport for reading/writing. + * + * @throws TTransportException if the transport could not be opened + */ + public abstract void open() + throws TTransportException; + + /** + * Closes the transport. + */ + public abstract void close(); + + /** + * Reads up to len bytes into buffer buf, starting att offset off. + * + * @param buf Array to read into + * @param off Index to start reading at + * @param len Maximum number of bytes to read + * @return The number of bytes actually read + * @throws TTransportException if there was an error reading data + */ + public abstract int read(byte[] buf, int off, int len) + throws TTransportException; + + /** + * Guarantees that all of len bytes are actually read off the transport. + * + * @param buf Array to read into + * @param off Index to start reading at + * @param len Maximum number of bytes to read + * @return The number of bytes actually read, which must be equal to len + * @throws TTransportException if there was an error reading data + */ + public int readAll(byte[] buf, int off, int len) + throws TTransportException { + int got = 0; + int ret = 0; + while (got < len) { + ret = read(buf, off+got, len-got); + if (ret <= 0) { + throw new TTransportException("Cannot read. Remote side has closed. Tried to read " + len + " bytes, but only got " + got + " bytes."); + } + got += ret; + } + return got; + } + + /** + * Writes the buffer to the output + * + * @param buf The output data buffer + * @throws TTransportException if an error occurs writing data + */ + public void write(byte[] buf) throws TTransportException { + write(buf, 0, buf.length); + } + + /** + * Writes up to len bytes from the buffer. + * + * @param buf The output data buffer + * @param off The offset to start writing from + * @param len The number of bytes to write + * @throws TTransportException if there was an error writing data + */ + public abstract void write(byte[] buf, int off, int len) + throws TTransportException; + + /** + * Flush any pending data out of a transport buffer. + * + * @throws TTransportException if there was an error writing out data. + */ + public void flush() + throws TTransportException {} +} diff --git a/lib/java/src/org/apache/thrift/transport/TTransportException.java b/lib/java/src/org/apache/thrift/transport/TTransportException.java new file mode 100644 index 00000000..d08f3b02 --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TTransportException.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import org.apache.thrift.TException; + +/** + * Transport exceptions. + * + */ +public class TTransportException extends TException { + + private static final long serialVersionUID = 1L; + + public static final int UNKNOWN = 0; + public static final int NOT_OPEN = 1; + public static final int ALREADY_OPEN = 2; + public static final int TIMED_OUT = 3; + public static final int END_OF_FILE = 4; + + protected int type_ = UNKNOWN; + + public TTransportException() { + super(); + } + + public TTransportException(int type) { + super(); + type_ = type; + } + + public TTransportException(int type, String message) { + super(message); + type_ = type; + } + + public TTransportException(String message) { + super(message); + } + + public TTransportException(int type, Throwable cause) { + super(cause); + type_ = type; + } + + public TTransportException(Throwable cause) { + super(cause); + } + + public TTransportException(String message, Throwable cause) { + super(message, cause); + } + + public TTransportException(int type, String message, Throwable cause) { + super(message, cause); + type_ = type; + } + + public int getType() { + return type_; + } + +} diff --git a/lib/java/src/org/apache/thrift/transport/TTransportFactory.java b/lib/java/src/org/apache/thrift/transport/TTransportFactory.java new file mode 100644 index 00000000..3e71630a --- /dev/null +++ b/lib/java/src/org/apache/thrift/transport/TTransportFactory.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +/** + * Factory class used to create wrapped instance of Transports. + * This is used primarily in servers, which get Transports from + * a ServerTransport and then may want to mutate them (i.e. create + * a BufferedTransport from the underlying base transport) + * + */ +public class TTransportFactory { + + /** + * Return a wrapped instance of the base Transport. + * + * @param trans The base transport + * @return Wrapped Transport + */ + public TTransport getTransport(TTransport trans) { + return trans; + } + +} diff --git a/lib/java/test/TestClient b/lib/java/test/TestClient new file mode 100755 index 00000000..bd3c996f --- /dev/null +++ b/lib/java/test/TestClient @@ -0,0 +1,22 @@ +#!/bin/bash -v + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +java -cp thrifttest.jar:../../lib/java/libthrift.jar org.apache.thrift.test.TestClient $* diff --git a/lib/java/test/TestNonblockingServer b/lib/java/test/TestNonblockingServer new file mode 100644 index 00000000..070991c7 --- /dev/null +++ b/lib/java/test/TestNonblockingServer @@ -0,0 +1,22 @@ +#!/bin/bash -v + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +java -server -Xmx256m -cp thrifttest.jar:../../lib/java/libthrift.jar org.apache.thrift.test.TestNonblockingServer $* diff --git a/lib/java/test/TestServer b/lib/java/test/TestServer new file mode 100755 index 00000000..0d36b58e --- /dev/null +++ b/lib/java/test/TestServer @@ -0,0 +1,22 @@ +#!/bin/bash -v + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +java -server -cp thrifttest.jar:../../lib/java/libthrift.jar org.apache.thrift.test.TestServer $* diff --git a/lib/java/test/org/apache/thrift/test/DeepCopyTest.java b/lib/java/test/org/apache/thrift/test/DeepCopyTest.java new file mode 100644 index 00000000..a171cab3 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/DeepCopyTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.thrift.test; + +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TSerializer; +import org.apache.thrift.protocol.TBinaryProtocol; +import thrift.test.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +public class DeepCopyTest { + + private static final byte[] kUnicodeBytes = { + (byte)0xd3, (byte)0x80, (byte)0xe2, (byte)0x85, (byte)0xae, (byte)0xce, + (byte)0x9d, (byte)0x20, (byte)0xd0, (byte)0x9d, (byte)0xce, (byte)0xbf, + (byte)0xe2, (byte)0x85, (byte)0xbf, (byte)0xd0, (byte)0xbe, (byte)0xc9, + (byte)0xa1, (byte)0xd0, (byte)0xb3, (byte)0xd0, (byte)0xb0, (byte)0xcf, + (byte)0x81, (byte)0xe2, (byte)0x84, (byte)0x8e, (byte)0x20, (byte)0xce, + (byte)0x91, (byte)0x74, (byte)0x74, (byte)0xce, (byte)0xb1, (byte)0xe2, + (byte)0x85, (byte)0xbd, (byte)0xce, (byte)0xba, (byte)0x83, (byte)0xe2, + (byte)0x80, (byte)0xbc + }; + + public static void main(String[] args) throws Exception { + TSerializer binarySerializer = new TSerializer(new TBinaryProtocol.Factory()); + TDeserializer binaryDeserializer = new TDeserializer(new TBinaryProtocol.Factory()); + + OneOfEach ooe = new OneOfEach(); + ooe.im_true = true; + ooe.im_false = false; + ooe.a_bite = (byte) 0xd6; + ooe.integer16 = 27000; + ooe.integer32 = 1 << 24; + ooe.integer64 = (long) 6000 * 1000 * 1000; + ooe.double_precision = Math.PI; + ooe.some_characters = "JSON THIS! \"\1"; + ooe.zomg_unicode = new String(kUnicodeBytes, "UTF-8"); + ooe.base64 = "string to bytes".getBytes(); + + Nesting n = new Nesting(new Bonk(), new OneOfEach()); + n.my_ooe.integer16 = 16; + n.my_ooe.integer32 = 32; + n.my_ooe.integer64 = 64; + n.my_ooe.double_precision = (Math.sqrt(5) + 1) / 2; + n.my_ooe.some_characters = ":R (me going \"rrrr\")"; + n.my_ooe.zomg_unicode = new String(kUnicodeBytes, "UTF-8"); + n.my_bonk.type = 31337; + n.my_bonk.message = "I am a bonk... xor!"; + + HolyMoley hm = new HolyMoley(); + + hm.big = new ArrayList(); + hm.big.add(ooe); + hm.big.add(n.my_ooe); + hm.big.get(0).a_bite = (byte) 0x22; + hm.big.get(1).a_bite = (byte) 0x23; + + hm.contain = new HashSet>(); + ArrayList stage1 = new ArrayList(2); + stage1.add("and a one"); + stage1.add("and a two"); + hm.contain.add(stage1); + stage1 = new ArrayList(3); + stage1.add("then a one, two"); + stage1.add("three!"); + stage1.add("FOUR!!"); + hm.contain.add(stage1); + stage1 = new ArrayList(0); + hm.contain.add(stage1); + + ArrayList stage2 = new ArrayList(); + hm.bonks = new HashMap>(); + hm.bonks.put("nothing", stage2); + Bonk b = new Bonk(); + b.type = 1; + b.message = "Wait."; + stage2.add(b); + b = new Bonk(); + b.type = 2; + b.message = "What?"; + stage2.add(b); + stage2 = new ArrayList(); + hm.bonks.put("something", stage2); + b = new Bonk(); + b.type = 3; + b.message = "quoth"; + b = new Bonk(); + b.type = 4; + b.message = "the raven"; + b = new Bonk(); + b.type = 5; + b.message = "nevermore"; + hm.bonks.put("poe", stage2); + + + byte[] binaryCopy = binarySerializer.serialize(hm); + HolyMoley hmCopy = new HolyMoley(); + binaryDeserializer.deserialize(hmCopy, binaryCopy); + HolyMoley hmCopy2 = new HolyMoley(hm); + + if (!hm.equals(hmCopy)) + throw new RuntimeException("copy constructor modified the original object!"); + if (!hmCopy.equals(hmCopy2)) + throw new RuntimeException("copy constructor generated incorrect copy"); + + hm.big.get(0).base64[0]++; // change binary value in original object + if (hm.equals(hmCopy2)) // make sure the change didn't propagate to the copied object + throw new RuntimeException("Binary field not copied correctly!"); + hm.big.get(0).base64[0]--; // undo change + + hmCopy2.bonks.get("nothing").get(1).message = "What else?"; + + if (hm.equals(hmCopy2)) + throw new RuntimeException("A deep copy was not done!"); + + } +} diff --git a/lib/java/test/org/apache/thrift/test/EqualityTest.java b/lib/java/test/org/apache/thrift/test/EqualityTest.java new file mode 100644 index 00000000..f01378f7 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/EqualityTest.java @@ -0,0 +1,661 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* +This program was generated by the following Python script: + +#!/usr/bin/python2.5 + +# Remove this when Python 2.6 hits the streets. +from __future__ import with_statement + +import sys +import os.path + + +# Quines the easy way. +with open(sys.argv[0], 'r') as handle: + source = handle.read() + +with open(os.path.join(os.path.dirname(sys.argv[0]), 'EqualityTest.java'), 'w') as out: + print >> out, ("/""*" r""" + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + """ "*""/") + print >> out + print >> out, "/""*" + print >> out, "This program was generated by the following Python script:" + print >> out + out.write(source) + print >> out, "*""/" + + print >> out, r''' +package org.apache.thrift.test; + +// Generated code +import thrift.test.*; + +/'''r'''** + *'''r'''/ +public class EqualityTest { + public static void main(String[] args) throws Exception { + JavaTestHelper lhs, rhs; +''' + + vals = { + 'int': ("1", "2"), + 'obj': ("\"foo\"", "\"bar\""), + 'bin': ("new byte[]{1,2}", "new byte[]{3,4}"), + } + matrix = ( + (False,False), + (False,True ), + (True ,False), + (True ,True ), + ) + + for type in ('int', 'obj', 'bin'): + for option in ('req', 'opt'): + nulls = matrix[0:1] if type == 'int' else matrix[-1::-1] + issets = matrix + for is_null in nulls: + for is_set in issets: + # isset is implied for non-primitives, so only consider the case + # where isset and non-null match. + if type != 'int' and list(is_set) != [ not null for null in is_null ]: + continue + for equal in (True, False): + print >> out + print >> out, " lhs = new JavaTestHelper();" + print >> out, " rhs = new JavaTestHelper();" + print >> out, " lhs." + option + "_" + type, "=", vals[type][0] + ";" + print >> out, " rhs." + option + "_" + type, "=", vals[type][0 if equal else 1] + ";" + isset_setter = "set" + option[0].upper() + option[1:] + "_" + type + "IsSet" + if (type == 'int' and is_set[0]): print >> out, " lhs." + isset_setter + "(true);" + if (type == 'int' and is_set[1]): print >> out, " rhs." + isset_setter + "(true);" + if (is_null[0]): print >> out, " lhs." + option + "_" + type, "= null;" + if (is_null[1]): print >> out, " rhs." + option + "_" + type, "= null;" + this_present = not is_null[0] and (option == 'req' or is_set[0]) + that_present = not is_null[1] and (option == 'req' or is_set[1]) + print >> out, " // this_present = " + repr(this_present) + print >> out, " // that_present = " + repr(that_present) + is_equal = \ + (not this_present and not that_present) or \ + (this_present and that_present and equal) + eq_str = 'true' if is_equal else 'false' + + print >> out, " if (lhs.equals(rhs) != "+eq_str+")" + print >> out, " throw new RuntimeException(\"Failure\");" + if is_equal: + print >> out, " if (lhs.hashCode() != rhs.hashCode())" + print >> out, " throw new RuntimeException(\"Failure\");" + + print >> out, r''' + } +} +''' +*/ + +package org.apache.thrift.test; + +// Generated code +import thrift.test.*; + +/** + */ +public class EqualityTest { + public static void main(String[] args) throws Exception { + JavaTestHelper lhs, rhs; + + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_int = 1; + rhs.req_int = 1; + // this_present = True + // that_present = True + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_int = 1; + rhs.req_int = 2; + // this_present = True + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_int = 1; + rhs.req_int = 1; + rhs.setReq_intIsSet(true); + // this_present = True + // that_present = True + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_int = 1; + rhs.req_int = 2; + rhs.setReq_intIsSet(true); + // this_present = True + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_int = 1; + rhs.req_int = 1; + lhs.setReq_intIsSet(true); + // this_present = True + // that_present = True + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_int = 1; + rhs.req_int = 2; + lhs.setReq_intIsSet(true); + // this_present = True + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_int = 1; + rhs.req_int = 1; + lhs.setReq_intIsSet(true); + rhs.setReq_intIsSet(true); + // this_present = True + // that_present = True + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_int = 1; + rhs.req_int = 2; + lhs.setReq_intIsSet(true); + rhs.setReq_intIsSet(true); + // this_present = True + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_int = 1; + rhs.opt_int = 1; + // this_present = False + // that_present = False + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_int = 1; + rhs.opt_int = 2; + // this_present = False + // that_present = False + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_int = 1; + rhs.opt_int = 1; + rhs.setOpt_intIsSet(true); + // this_present = False + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_int = 1; + rhs.opt_int = 2; + rhs.setOpt_intIsSet(true); + // this_present = False + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_int = 1; + rhs.opt_int = 1; + lhs.setOpt_intIsSet(true); + // this_present = True + // that_present = False + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_int = 1; + rhs.opt_int = 2; + lhs.setOpt_intIsSet(true); + // this_present = True + // that_present = False + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_int = 1; + rhs.opt_int = 1; + lhs.setOpt_intIsSet(true); + rhs.setOpt_intIsSet(true); + // this_present = True + // that_present = True + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_int = 1; + rhs.opt_int = 2; + lhs.setOpt_intIsSet(true); + rhs.setOpt_intIsSet(true); + // this_present = True + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_obj = "foo"; + rhs.req_obj = "foo"; + lhs.req_obj = null; + rhs.req_obj = null; + // this_present = False + // that_present = False + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_obj = "foo"; + rhs.req_obj = "bar"; + lhs.req_obj = null; + rhs.req_obj = null; + // this_present = False + // that_present = False + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_obj = "foo"; + rhs.req_obj = "foo"; + lhs.req_obj = null; + // this_present = False + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_obj = "foo"; + rhs.req_obj = "bar"; + lhs.req_obj = null; + // this_present = False + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_obj = "foo"; + rhs.req_obj = "foo"; + rhs.req_obj = null; + // this_present = True + // that_present = False + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_obj = "foo"; + rhs.req_obj = "bar"; + rhs.req_obj = null; + // this_present = True + // that_present = False + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_obj = "foo"; + rhs.req_obj = "foo"; + // this_present = True + // that_present = True + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_obj = "foo"; + rhs.req_obj = "bar"; + // this_present = True + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_obj = "foo"; + rhs.opt_obj = "foo"; + lhs.opt_obj = null; + rhs.opt_obj = null; + // this_present = False + // that_present = False + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_obj = "foo"; + rhs.opt_obj = "bar"; + lhs.opt_obj = null; + rhs.opt_obj = null; + // this_present = False + // that_present = False + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_obj = "foo"; + rhs.opt_obj = "foo"; + lhs.opt_obj = null; + // this_present = False + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_obj = "foo"; + rhs.opt_obj = "bar"; + lhs.opt_obj = null; + // this_present = False + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_obj = "foo"; + rhs.opt_obj = "foo"; + rhs.opt_obj = null; + // this_present = True + // that_present = False + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_obj = "foo"; + rhs.opt_obj = "bar"; + rhs.opt_obj = null; + // this_present = True + // that_present = False + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_obj = "foo"; + rhs.opt_obj = "foo"; + // this_present = True + // that_present = True + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_obj = "foo"; + rhs.opt_obj = "bar"; + // this_present = True + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_bin = new byte[]{1,2}; + rhs.req_bin = new byte[]{1,2}; + lhs.req_bin = null; + rhs.req_bin = null; + // this_present = False + // that_present = False + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_bin = new byte[]{1,2}; + rhs.req_bin = new byte[]{3,4}; + lhs.req_bin = null; + rhs.req_bin = null; + // this_present = False + // that_present = False + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_bin = new byte[]{1,2}; + rhs.req_bin = new byte[]{1,2}; + lhs.req_bin = null; + // this_present = False + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_bin = new byte[]{1,2}; + rhs.req_bin = new byte[]{3,4}; + lhs.req_bin = null; + // this_present = False + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_bin = new byte[]{1,2}; + rhs.req_bin = new byte[]{1,2}; + rhs.req_bin = null; + // this_present = True + // that_present = False + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_bin = new byte[]{1,2}; + rhs.req_bin = new byte[]{3,4}; + rhs.req_bin = null; + // this_present = True + // that_present = False + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_bin = new byte[]{1,2}; + rhs.req_bin = new byte[]{1,2}; + // this_present = True + // that_present = True + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.req_bin = new byte[]{1,2}; + rhs.req_bin = new byte[]{3,4}; + // this_present = True + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_bin = new byte[]{1,2}; + rhs.opt_bin = new byte[]{1,2}; + lhs.opt_bin = null; + rhs.opt_bin = null; + // this_present = False + // that_present = False + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_bin = new byte[]{1,2}; + rhs.opt_bin = new byte[]{3,4}; + lhs.opt_bin = null; + rhs.opt_bin = null; + // this_present = False + // that_present = False + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_bin = new byte[]{1,2}; + rhs.opt_bin = new byte[]{1,2}; + lhs.opt_bin = null; + // this_present = False + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_bin = new byte[]{1,2}; + rhs.opt_bin = new byte[]{3,4}; + lhs.opt_bin = null; + // this_present = False + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_bin = new byte[]{1,2}; + rhs.opt_bin = new byte[]{1,2}; + rhs.opt_bin = null; + // this_present = True + // that_present = False + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_bin = new byte[]{1,2}; + rhs.opt_bin = new byte[]{3,4}; + rhs.opt_bin = null; + // this_present = True + // that_present = False + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_bin = new byte[]{1,2}; + rhs.opt_bin = new byte[]{1,2}; + // this_present = True + // that_present = True + if (lhs.equals(rhs) != true) + throw new RuntimeException("Failure"); + if (lhs.hashCode() != rhs.hashCode()) + throw new RuntimeException("Failure"); + + lhs = new JavaTestHelper(); + rhs = new JavaTestHelper(); + lhs.opt_bin = new byte[]{1,2}; + rhs.opt_bin = new byte[]{3,4}; + // this_present = True + // that_present = True + if (lhs.equals(rhs) != false) + throw new RuntimeException("Failure"); + + } +} + diff --git a/lib/java/test/org/apache/thrift/test/Fixtures.java b/lib/java/test/org/apache/thrift/test/Fixtures.java new file mode 100644 index 00000000..14ac44f7 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/Fixtures.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.thrift.test; + +import java.util.*; +import thrift.test.*; + +public class Fixtures { + + private static final byte[] kUnicodeBytes = { + (byte)0xd3, (byte)0x80, (byte)0xe2, (byte)0x85, (byte)0xae, (byte)0xce, + (byte)0x9d, (byte)0x20, (byte)0xd0, (byte)0x9d, (byte)0xce, (byte)0xbf, + (byte)0xe2, (byte)0x85, (byte)0xbf, (byte)0xd0, (byte)0xbe, (byte)0xc9, + (byte)0xa1, (byte)0xd0, (byte)0xb3, (byte)0xd0, (byte)0xb0, (byte)0xcf, + (byte)0x81, (byte)0xe2, (byte)0x84, (byte)0x8e, (byte)0x20, (byte)0xce, + (byte)0x91, (byte)0x74, (byte)0x74, (byte)0xce, (byte)0xb1, (byte)0xe2, + (byte)0x85, (byte)0xbd, (byte)0xce, (byte)0xba, (byte)0x83, (byte)0xe2, + (byte)0x80, (byte)0xbc + }; + + + public static final OneOfEach oneOfEach; + public static final Nesting nesting; + public static final HolyMoley holyMoley; + public static final CompactProtoTestStruct compactProtoTestStruct; + + static { + try { + oneOfEach = new OneOfEach(); + oneOfEach.im_true = true; + oneOfEach.im_false = false; + oneOfEach.a_bite = (byte) 0x03; + oneOfEach.integer16 = 27000; + oneOfEach.integer32 = 1 << 24; + oneOfEach.integer64 = (long) 6000 * 1000 * 1000; + oneOfEach.double_precision = Math.PI; + oneOfEach.some_characters = "JSON THIS! \"\1"; + oneOfEach.zomg_unicode = new String(kUnicodeBytes, "UTF-8"); + + nesting = new Nesting(new Bonk(), new OneOfEach()); + nesting.my_ooe.integer16 = 16; + nesting.my_ooe.integer32 = 32; + nesting.my_ooe.integer64 = 64; + nesting.my_ooe.double_precision = (Math.sqrt(5) + 1) / 2; + nesting.my_ooe.some_characters = ":R (me going \"rrrr\")"; + nesting.my_ooe.zomg_unicode = new String(kUnicodeBytes, "UTF-8"); + nesting.my_bonk.type = 31337; + nesting.my_bonk.message = "I am a bonk... xor!"; + + holyMoley = new HolyMoley(); + + holyMoley.big = new ArrayList(); + holyMoley.big.add(new OneOfEach(oneOfEach)); + holyMoley.big.add(nesting.my_ooe); + holyMoley.big.get(0).a_bite = (byte) 0x22; + holyMoley.big.get(1).a_bite = (byte) 0x23; + + holyMoley.contain = new HashSet>(); + ArrayList stage1 = new ArrayList(2); + stage1.add("and a one"); + stage1.add("and a two"); + holyMoley.contain.add(stage1); + stage1 = new ArrayList(3); + stage1.add("then a one, two"); + stage1.add("three!"); + stage1.add("FOUR!!"); + holyMoley.contain.add(stage1); + stage1 = new ArrayList(0); + holyMoley.contain.add(stage1); + + ArrayList stage2 = new ArrayList(); + holyMoley.bonks = new HashMap>(); + // one empty + holyMoley.bonks.put("nothing", stage2); + + // one with two + stage2 = new ArrayList(); + Bonk b = new Bonk(); + b.type = 1; + b.message = "Wait."; + stage2.add(b); + b = new Bonk(); + b.type = 2; + b.message = "What?"; + stage2.add(b); + holyMoley.bonks.put("something", stage2); + + // one with three + stage2 = new ArrayList(); + b = new Bonk(); + b.type = 3; + b.message = "quoth"; + b = new Bonk(); + b.type = 4; + b.message = "the raven"; + b = new Bonk(); + b.type = 5; + b.message = "nevermore"; + holyMoley.bonks.put("poe", stage2); + + // superhuge compact proto test struct + compactProtoTestStruct = new CompactProtoTestStruct(thrift.test.Constants.COMPACT_TEST); + compactProtoTestStruct.a_binary = new byte[]{0,1,2,3,4,5,6,7,8}; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/lib/java/test/org/apache/thrift/test/IdentityTest.java b/lib/java/test/org/apache/thrift/test/IdentityTest.java new file mode 100644 index 00000000..c6453ce7 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/IdentityTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.test; + +// Generated code +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TSerializer; +import org.apache.thrift.protocol.TBinaryProtocol; + +import thrift.test.Bonk; +import thrift.test.HolyMoley; +import thrift.test.Nesting; +import thrift.test.OneOfEach; + +/** + * + */ +public class IdentityTest { + public static Object deepCopy(Object oldObj) throws Exception { + ObjectOutputStream oos = null; + ObjectInputStream ois = null; + try { + ByteArrayOutputStream bos = + new ByteArrayOutputStream(); + oos = new ObjectOutputStream(bos); + oos.writeObject(oldObj); + oos.flush(); + ByteArrayInputStream bis = + new ByteArrayInputStream(bos.toByteArray()); + ois = new ObjectInputStream(bis); + return ois.readObject(); + } finally { + oos.close(); + ois.close(); + } + } + + public static void main(String[] args) throws Exception { + TSerializer binarySerializer = new TSerializer(new TBinaryProtocol.Factory()); + TDeserializer binaryDeserializer = new TDeserializer(new TBinaryProtocol.Factory()); + + OneOfEach ooe = new OneOfEach(); + ooe.im_true = true; + ooe.im_false = false; + ooe.a_bite = (byte)0xd6; + ooe.integer16 = 27000; + ooe.integer32 = 1<<24; + ooe.integer64 = (long)6000 * 1000 * 1000; + ooe.double_precision = Math.PI; + ooe.some_characters = "JSON THIS! \"\u0001"; + ooe.base64 = new byte[]{1,2,3,(byte)255}; + + Nesting n = new Nesting(); + n.my_ooe = (OneOfEach)deepCopy(ooe); + n.my_ooe.integer16 = 16; + n.my_ooe.integer32 = 32; + n.my_ooe.integer64 = 64; + n.my_ooe.double_precision = (Math.sqrt(5)+1)/2; + n.my_ooe.some_characters = ":R (me going \"rrrr\")"; + n.my_ooe.zomg_unicode = "\u04c0\u216e\u039d\u0020\u041d\u03bf\u217f"+ + "\u043e\u0261\u0433\u0430\u03c1\u210e\u0020"+ + "\u0391\u0074\u0074\u03b1\u217d\u03ba\u01c3"+ + "\u203c"; + n.my_bonk = new Bonk(); + n.my_bonk.type = 31337; + n.my_bonk.message = "I am a bonk... xor!"; + + HolyMoley hm = new HolyMoley(); + hm.big = new ArrayList(); + hm.contain = new HashSet>(); + hm.bonks = new HashMap>(); + + hm.big.add((OneOfEach)deepCopy(ooe)); + hm.big.add((OneOfEach)deepCopy(n.my_ooe)); + hm.big.get(0).a_bite = 0x22; + hm.big.get(1).a_bite = 0x33; + + List stage1 = new ArrayList(); + stage1.add("and a one"); + stage1.add("and a two"); + hm.contain.add(stage1); + stage1 = new ArrayList(); + stage1.add("then a one, two"); + stage1.add("three!"); + stage1.add("FOUR!!"); + hm.contain.add(stage1); + stage1 = new ArrayList(); + hm.contain.add(stage1); + + List stage2 = new ArrayList(); + hm.bonks.put("nothing", stage2); + stage2.add(new Bonk()); + stage2.get(0).type = 1; + stage2.get(0).message = "Wait."; + stage2.add(new Bonk()); + stage2.get(1).type = 2; + stage2.get(1).message = "What?"; + hm.bonks.put("something", stage2); + stage2 = new ArrayList(); + stage2.add(new Bonk()); + stage2.get(0).type = 3; + stage2.get(0).message = "quoth"; + stage2.add(new Bonk()); + stage2.get(1).type = 4; + stage2.get(1).message = "the raven"; + stage2.add(new Bonk()); + stage2.get(2).type = 5; + stage2.get(2).message = "nevermore"; + hm.bonks.put("poe", stage2); + + OneOfEach ooe2 = new OneOfEach(); + binaryDeserializer.deserialize( + ooe2, + binarySerializer.serialize(ooe)); + + if (!ooe.equals(ooe2)) { + throw new RuntimeException("Failure: ooe (equals)"); + } + if (ooe.hashCode() != ooe2.hashCode()) { + throw new RuntimeException("Failure: ooe (hash)"); + } + + + Nesting n2 = new Nesting(); + binaryDeserializer.deserialize( + n2, + binarySerializer.serialize(n)); + + if (!n.equals(n2)) { + throw new RuntimeException("Failure: n (equals)"); + } + if (n.hashCode() != n2.hashCode()) { + throw new RuntimeException("Failure: n (hash)"); + } + + HolyMoley hm2 = new HolyMoley(); + binaryDeserializer.deserialize( + hm2, + binarySerializer.serialize(hm)); + + if (!hm.equals(hm2)) { + throw new RuntimeException("Failure: hm (equals)"); + } + if (hm.hashCode() != hm2.hashCode()) { + throw new RuntimeException("Failure: hm (hash)"); + } + + } +} diff --git a/lib/java/test/org/apache/thrift/test/JSONProtoTest.java b/lib/java/test/org/apache/thrift/test/JSONProtoTest.java new file mode 100644 index 00000000..59f4ce18 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/JSONProtoTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.test; + +// Generated code +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +import org.apache.thrift.protocol.TJSONProtocol; +import org.apache.thrift.transport.TMemoryBuffer; + +import thrift.test.Base64; +import thrift.test.Bonk; +import thrift.test.HolyMoley; +import thrift.test.Nesting; +import thrift.test.OneOfEach; + +/** + * Tests for the Java implementation of TJSONProtocol. Mirrors the C++ version + * + */ +public class JSONProtoTest { + + private static final byte[] kUnicodeBytes = { + (byte)0xd3, (byte)0x80, (byte)0xe2, (byte)0x85, (byte)0xae, (byte)0xce, + (byte)0x9d, (byte)0x20, (byte)0xd0, (byte)0x9d, (byte)0xce, (byte)0xbf, + (byte)0xe2, (byte)0x85, (byte)0xbf, (byte)0xd0, (byte)0xbe, (byte)0xc9, + (byte)0xa1, (byte)0xd0, (byte)0xb3, (byte)0xd0, (byte)0xb0, (byte)0xcf, + (byte)0x81, (byte)0xe2, (byte)0x84, (byte)0x8e, (byte)0x20, (byte)0xce, + (byte)0x91, (byte)0x74, (byte)0x74, (byte)0xce, (byte)0xb1, (byte)0xe2, + (byte)0x85, (byte)0xbd, (byte)0xce, (byte)0xba, (byte)0x83, (byte)0xe2, + (byte)0x80, (byte)0xbc + }; + + public static void main(String [] args) throws Exception { + try { + System.out.println("In JSON Proto test"); + + OneOfEach ooe = new OneOfEach(); + ooe.im_true = true; + ooe.im_false = false; + ooe.a_bite = (byte)0xd6; + ooe.integer16 = 27000; + ooe.integer32 = 1<<24; + ooe.integer64 = (long)6000 * 1000 * 1000; + ooe.double_precision = Math.PI; + ooe.some_characters = "JSON THIS! \"\1"; + ooe.zomg_unicode = new String(kUnicodeBytes, "UTF-8"); + + + Nesting n = new Nesting(new Bonk(), new OneOfEach()); + n.my_ooe.integer16 = 16; + n.my_ooe.integer32 = 32; + n.my_ooe.integer64 = 64; + n.my_ooe.double_precision = (Math.sqrt(5)+1)/2; + n.my_ooe.some_characters = ":R (me going \"rrrr\")"; + n.my_ooe.zomg_unicode = new String(kUnicodeBytes, "UTF-8"); + n.my_bonk.type = 31337; + n.my_bonk.message = "I am a bonk... xor!"; + + HolyMoley hm = new HolyMoley(); + + hm.big = new ArrayList(); + hm.big.add(ooe); + hm.big.add(n.my_ooe); + hm.big.get(0).a_bite = (byte)0x22; + hm.big.get(1).a_bite = (byte)0x23; + + hm.contain = new HashSet>(); + ArrayList stage1 = new ArrayList(2); + stage1.add("and a one"); + stage1.add("and a two"); + hm.contain.add(stage1); + stage1 = new ArrayList(3); + stage1.add("then a one, two"); + stage1.add("three!"); + stage1.add("FOUR!!"); + hm.contain.add(stage1); + stage1 = new ArrayList(0); + hm.contain.add(stage1); + + ArrayList stage2 = new ArrayList(); + hm.bonks = new HashMap>(); + hm.bonks.put("nothing", stage2); + Bonk b = new Bonk(); + b.type = 1; + b.message = "Wait."; + stage2.add(b); + b = new Bonk(); + b.type = 2; + b.message = "What?"; + stage2.add(b); + stage2 = new ArrayList(); + hm.bonks.put("something", stage2); + b = new Bonk(); + b.type = 3; + b.message = "quoth"; + b = new Bonk(); + b.type = 4; + b.message = "the raven"; + b = new Bonk(); + b.type = 5; + b.message = "nevermore"; + hm.bonks.put("poe", stage2); + + TMemoryBuffer buffer = new TMemoryBuffer(1024); + TJSONProtocol proto = new TJSONProtocol(buffer); + + System.out.println("Writing ooe"); + ooe.write(proto); + System.out.println("Reading ooe"); + OneOfEach ooe2 = new OneOfEach(); + ooe2.read(proto); + + System.out.println("Comparing ooe"); + if (!ooe.equals(ooe2)) { + throw new RuntimeException("ooe != ooe2"); + } + + System.out.println("Writing hm"); + hm.write(proto); + + System.out.println("Reading hm"); + HolyMoley hm2 = new HolyMoley(); + hm2.read(proto); + + System.out.println("Comparing hm"); + if (!hm.equals(hm2)) { + throw new RuntimeException("hm != hm2"); + } + + hm2.big.get(0).a_bite = (byte)0xFF; + if (hm.equals(hm2)) { + throw new RuntimeException("hm should not equal hm2"); + } + + Base64 base = new Base64(); + base.a = 123; + base.b1 = "1".getBytes("UTF-8"); + base.b2 = "12".getBytes("UTF-8"); + base.b3 = "123".getBytes("UTF-8"); + base.b4 = "1234".getBytes("UTF-8"); + base.b5 = "12345".getBytes("UTF-8"); + base.b6 = "123456".getBytes("UTF-8"); + + System.out.println("Writing base"); + base.write(proto); + + System.out.println("Reading base"); + Base64 base2 = new Base64(); + base2.read(proto); + + System.out.println("Comparing base"); + if (!base.equals(base2)) { + throw new RuntimeException("base != base2"); + } + + } catch (Exception ex) { + ex.printStackTrace(); + throw ex; + } + } + +} diff --git a/lib/java/test/org/apache/thrift/test/JavaBeansTest.java b/lib/java/test/org/apache/thrift/test/JavaBeansTest.java new file mode 100644 index 00000000..b72bd388 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/JavaBeansTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.test; + +import java.util.LinkedList; +import thrift.test.OneOfEachBeans; + +public class JavaBeansTest { + public static void main(String[] args) throws Exception { + // Test isSet methods + OneOfEachBeans ooe = new OneOfEachBeans(); + + // Nothing should be set + if (ooe.is_set_a_bite()) + throw new RuntimeException("isSet method error: unset field returned as set!"); + if (ooe.is_set_base64()) + throw new RuntimeException("isSet method error: unset field returned as set!"); + if (ooe.is_set_byte_list()) + throw new RuntimeException("isSet method error: unset field returned as set!"); + if (ooe.is_set_double_precision()) + throw new RuntimeException("isSet method error: unset field returned as set!"); + if (ooe.is_set_i16_list()) + throw new RuntimeException("isSet method error: unset field returned as set!"); + if (ooe.is_set_i64_list()) + throw new RuntimeException("isSet method error: unset field returned as set!"); + if (ooe.is_set_boolean_field()) + throw new RuntimeException("isSet method error: unset field returned as set!"); + if (ooe.is_set_integer16()) + throw new RuntimeException("isSet method error: unset field returned as set!"); + if (ooe.is_set_integer32()) + throw new RuntimeException("isSet method error: unset field returned as set!"); + if (ooe.is_set_integer64()) + throw new RuntimeException("isSet method error: unset field returned as set!"); + if (ooe.is_set_some_characters()) + throw new RuntimeException("isSet method error: unset field returned as set!"); + + for (int i = 1; i < 12; i++){ + if (ooe.isSet(i)) + throw new RuntimeException("isSet method error: unset field " + i + " returned as set!"); + } + + // Everything is set + ooe.set_a_bite((byte) 1); + ooe.set_base64("bytes".getBytes()); + ooe.set_byte_list(new LinkedList()); + ooe.set_double_precision(1); + ooe.set_i16_list(new LinkedList()); + ooe.set_i64_list(new LinkedList()); + ooe.set_boolean_field(true); + ooe.set_integer16((short) 1); + ooe.set_integer32(1); + ooe.set_integer64(1); + ooe.set_some_characters("string"); + + if (!ooe.is_set_a_bite()) + throw new RuntimeException("isSet method error: set field returned as unset!"); + if (!ooe.is_set_base64()) + throw new RuntimeException("isSet method error: set field returned as unset!"); + if (!ooe.is_set_byte_list()) + throw new RuntimeException("isSet method error: set field returned as unset!"); + if (!ooe.is_set_double_precision()) + throw new RuntimeException("isSet method error: set field returned as unset!"); + if (!ooe.is_set_i16_list()) + throw new RuntimeException("isSet method error: set field returned as unset!"); + if (!ooe.is_set_i64_list()) + throw new RuntimeException("isSet method error: set field returned as unset!"); + if (!ooe.is_set_boolean_field()) + throw new RuntimeException("isSet method error: set field returned as unset!"); + if (!ooe.is_set_integer16()) + throw new RuntimeException("isSet method error: set field returned as unset!"); + if (!ooe.is_set_integer32()) + throw new RuntimeException("isSet method error: set field returned as unset!"); + if (!ooe.is_set_integer64()) + throw new RuntimeException("isSet method error: set field returned as unset!"); + if (!ooe.is_set_some_characters()) + throw new RuntimeException("isSet method error: set field returned as unset!"); + + for (int i = 1; i < 12; i++){ + if (!ooe.isSet(i)) + throw new RuntimeException("isSet method error: set field " + i + " returned as unset!"); + } + + // Should throw exception when field doesn't exist + boolean exceptionThrown = false; + try{ + if (ooe.isSet(100)); + } catch (IllegalArgumentException e){ + exceptionThrown = true; + } + if (!exceptionThrown) + throw new RuntimeException("isSet method error: non-existent field provided as agument but no exception thrown!"); + } +} diff --git a/lib/java/test/org/apache/thrift/test/MetaDataTest.java b/lib/java/test/org/apache/thrift/test/MetaDataTest.java new file mode 100644 index 00000000..a0180348 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/MetaDataTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.thrift.test; + +import java.util.Map; +import org.apache.thrift.TFieldRequirementType; +import org.apache.thrift.meta_data.FieldMetaData; +import org.apache.thrift.meta_data.ListMetaData; +import org.apache.thrift.meta_data.MapMetaData; +import org.apache.thrift.meta_data.SetMetaData; +import org.apache.thrift.meta_data.StructMetaData; +import org.apache.thrift.protocol.TType; +import thrift.test.*; + +public class MetaDataTest { + + public static void main(String[] args) throws Exception { + Map mdMap = CrazyNesting.metaDataMap; + + // Check for struct fields existence + if (mdMap.size() != 3) + throw new RuntimeException("metadata map contains wrong number of entries!"); + if (!mdMap.containsKey(CrazyNesting.SET_FIELD) || !mdMap.containsKey(CrazyNesting.LIST_FIELD) || !mdMap.containsKey(CrazyNesting.STRING_FIELD)) + throw new RuntimeException("metadata map doesn't contain entry for a struct field!"); + + // Check for struct fields contents + if (!mdMap.get(CrazyNesting.STRING_FIELD).fieldName.equals("string_field") || + !mdMap.get(CrazyNesting.LIST_FIELD).fieldName.equals("list_field") || + !mdMap.get(CrazyNesting.SET_FIELD).fieldName.equals("set_field")) + throw new RuntimeException("metadata map contains a wrong fieldname"); + if (mdMap.get(CrazyNesting.STRING_FIELD).requirementType != TFieldRequirementType.DEFAULT || + mdMap.get(CrazyNesting.LIST_FIELD).requirementType != TFieldRequirementType.REQUIRED || + mdMap.get(CrazyNesting.SET_FIELD).requirementType != TFieldRequirementType.OPTIONAL) + throw new RuntimeException("metadata map contains the wrong requirement type for a field"); + if (mdMap.get(CrazyNesting.STRING_FIELD).valueMetaData.type != TType.STRING || + mdMap.get(CrazyNesting.LIST_FIELD).valueMetaData.type != TType.LIST || + mdMap.get(CrazyNesting.SET_FIELD).valueMetaData.type != TType.SET) + throw new RuntimeException("metadata map contains the wrong requirement type for a field"); + + // Check nested structures + if (!mdMap.get(CrazyNesting.LIST_FIELD).valueMetaData.isContainer()) + throw new RuntimeException("value metadata for a list is stored as non-container!"); + if (mdMap.get(CrazyNesting.LIST_FIELD).valueMetaData.isStruct()) + throw new RuntimeException("value metadata for a list is stored as a struct!"); + if (((MapMetaData)((ListMetaData)((SetMetaData)((MapMetaData)((MapMetaData)((ListMetaData)mdMap.get(CrazyNesting.LIST_FIELD).valueMetaData).elemMetaData).valueMetaData).valueMetaData).elemMetaData).elemMetaData).keyMetaData.type != TType.STRUCT) + throw new RuntimeException("metadata map contains wrong type for a value in a deeply nested structure"); + if (((StructMetaData)((MapMetaData)((ListMetaData)((SetMetaData)((MapMetaData)((MapMetaData)((ListMetaData)mdMap.get(CrazyNesting.LIST_FIELD).valueMetaData).elemMetaData).valueMetaData).valueMetaData).elemMetaData).elemMetaData).keyMetaData).structClass != Insanity.class) + throw new RuntimeException("metadata map contains wrong class for a struct in a deeply nested structure"); + + // Check that FieldMetaData contains a map with metadata for all generated struct classes + if (FieldMetaData.getStructMetaDataMap(CrazyNesting.class) == null || + FieldMetaData.getStructMetaDataMap(Insanity.class) == null || + FieldMetaData.getStructMetaDataMap(Xtruct.class) == null) + throw new RuntimeException("global metadata map doesn't contain an entry for a known struct"); + if (FieldMetaData.getStructMetaDataMap(CrazyNesting.class) != CrazyNesting.metaDataMap || + FieldMetaData.getStructMetaDataMap(Insanity.class) != Insanity.metaDataMap) + throw new RuntimeException("global metadata map contains wrong entry for a loaded struct"); + } +} diff --git a/lib/java/test/org/apache/thrift/test/OverloadNonblockingServer.java b/lib/java/test/org/apache/thrift/test/OverloadNonblockingServer.java new file mode 100644 index 00000000..54d78e56 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/OverloadNonblockingServer.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.thrift.test; + +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.transport.TSocket; + + +public class OverloadNonblockingServer { + + public static void main(String[] args) throws Exception { + int msg_size_mb = Integer.parseInt(args[0]); + int msg_size = msg_size_mb * 1024 * 1024; + + TSocket socket = new TSocket("localhost", 9090); + TBinaryProtocol binprot = new TBinaryProtocol(socket); + socket.open(); + binprot.writeI32(msg_size); + binprot.writeI32(1); + socket.flush(); + + System.in.read(); + // Thread.sleep(30000); + for (int i = 0; i < msg_size_mb; i++) { + binprot.writeBinary(new byte[1024 * 1024]); + } + + socket.close(); + } +} diff --git a/lib/java/test/org/apache/thrift/test/ReadStruct.java b/lib/java/test/org/apache/thrift/test/ReadStruct.java new file mode 100644 index 00000000..2dc042c5 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/ReadStruct.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.test; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; + +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.thrift.transport.TTransport; + +import thrift.test.CompactProtoTestStruct; + +public class ReadStruct { + public static void main(String[] args) throws Exception { + if (args.length != 2) { + System.out.println("usage: java -cp build/classes org.apache.thrift.test.ReadStruct filename proto_factory_class"); + System.out.println("Read in an instance of CompactProtocolTestStruct from 'file', making sure that it is equivalent to Fixtures.compactProtoTestStruct. Use a protocol from 'proto_factory_class'."); + } + + TTransport trans = new TIOStreamTransport(new BufferedInputStream(new FileInputStream(args[0]))); + + TProtocolFactory factory = (TProtocolFactory)Class.forName(args[1]).newInstance(); + + TProtocol proto = factory.getProtocol(trans); + + CompactProtoTestStruct cpts = new CompactProtoTestStruct(); + + for (Integer fid : CompactProtoTestStruct.metaDataMap.keySet()) { + cpts.setFieldValue(fid, null); + } + + cpts.read(proto); + + if (cpts.equals(Fixtures.compactProtoTestStruct)) { + System.out.println("Object verified successfully!"); + } else { + System.out.println("Object failed verification!"); + System.out.println("Expected: " + Fixtures.compactProtoTestStruct + " but got " + cpts); + } + + } + +} diff --git a/lib/java/test/org/apache/thrift/test/SerializationBenchmark.java b/lib/java/test/org/apache/thrift/test/SerializationBenchmark.java new file mode 100644 index 00000000..b83b2f9b --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/SerializationBenchmark.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.thrift.test; + +import java.io.ByteArrayInputStream; + +import org.apache.thrift.*; +import org.apache.thrift.protocol.*; +import org.apache.thrift.transport.*; + +import thrift.test.*; + +public class SerializationBenchmark { + private final static int HOW_MANY = 10000000; + + public static void main(String[] args) throws Exception { + TProtocolFactory factory = new TBinaryProtocol.Factory(); + + OneOfEach ooe = new OneOfEach(); + ooe.im_true = true; + ooe.im_false = false; + ooe.a_bite = (byte)0xd6; + ooe.integer16 = 27000; + ooe.integer32 = 1<<24; + ooe.integer64 = (long)6000 * 1000 * 1000; + ooe.double_precision = Math.PI; + ooe.some_characters = "JSON THIS! \"\u0001"; + ooe.base64 = new byte[]{1,2,3,(byte)255}; + + testSerialization(factory, ooe); + testDeserialization(factory, ooe, OneOfEach.class); + } + + public static void testSerialization(TProtocolFactory factory, TBase object) throws Exception { + TTransport trans = new TTransport() { + public void write(byte[] bin, int x, int y) throws TTransportException {} + public int read(byte[] bin, int x, int y) throws TTransportException {return 0;} + public void close() {} + public void open() {} + public boolean isOpen() {return true;} + }; + + TProtocol proto = factory.getProtocol(trans); + + long startTime = System.currentTimeMillis(); + for (int i = 0; i < HOW_MANY; i++) { + object.write(proto); + } + long endTime = System.currentTimeMillis(); + + System.out.println("Test time: " + (endTime - startTime) + " ms"); + } + + public static void testDeserialization(TProtocolFactory factory, T object, Class klass) throws Exception { + TMemoryBuffer buf = new TMemoryBuffer(0); + object.write(factory.getProtocol(buf)); + byte[] serialized = new byte[100*1024]; + buf.read(serialized, 0, 100*1024); + + long startTime = System.currentTimeMillis(); + for (int i = 0; i < HOW_MANY; i++) { + T o2 = klass.newInstance(); + o2.read(factory.getProtocol(new TIOStreamTransport(new ByteArrayInputStream(serialized)))); + } + long endTime = System.currentTimeMillis(); + + System.out.println("Test time: " + (endTime - startTime) + " ms"); + } +} \ No newline at end of file diff --git a/lib/java/test/org/apache/thrift/test/TCompactProtocolTest.java b/lib/java/test/org/apache/thrift/test/TCompactProtocolTest.java new file mode 100755 index 00000000..5d1c3cba --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/TCompactProtocolTest.java @@ -0,0 +1,450 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.thrift.test; + +import java.util.Arrays; +import java.util.List; + +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.protocol.TField; +import org.apache.thrift.protocol.TMessage; +import org.apache.thrift.protocol.TMessageType; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.protocol.TStruct; +import org.apache.thrift.protocol.TType; +import org.apache.thrift.transport.TMemoryBuffer; + +import thrift.test.CompactProtoTestStruct; +import thrift.test.HolyMoley; +import thrift.test.Nesting; +import thrift.test.OneOfEach; +import thrift.test.Srv; + +public class TCompactProtocolTest { + + static TProtocolFactory factory = new TCompactProtocol.Factory(); + + public static void main(String[] args) throws Exception { + testNakedByte(); + for (int i = 0; i < 128; i++) { + testByteField((byte)i); + testByteField((byte)-i); + } + + testNakedI16((short)0); + testNakedI16((short)1); + testNakedI16((short)15000); + testNakedI16((short)0x7fff); + testNakedI16((short)-1); + testNakedI16((short)-15000); + testNakedI16((short)-0x7fff); + + testI16Field((short)0); + testI16Field((short)1); + testI16Field((short)7); + testI16Field((short)150); + testI16Field((short)15000); + testI16Field((short)0x7fff); + testI16Field((short)-1); + testI16Field((short)-7); + testI16Field((short)-150); + testI16Field((short)-15000); + testI16Field((short)-0x7fff); + + testNakedI32(0); + testNakedI32(1); + testNakedI32(15000); + testNakedI32(0xffff); + testNakedI32(-1); + testNakedI32(-15000); + testNakedI32(-0xffff); + + testI32Field(0); + testI32Field(1); + testI32Field(7); + testI32Field(150); + testI32Field(15000); + testI32Field(31337); + testI32Field(0xffff); + testI32Field(0xffffff); + testI32Field(-1); + testI32Field(-7); + testI32Field(-150); + testI32Field(-15000); + testI32Field(-0xffff); + testI32Field(-0xffffff); + + testNakedI64(0); + for (int i = 0; i < 62; i++) { + testNakedI64(1L << i); + testNakedI64(-(1L << i)); + } + + testI64Field(0); + for (int i = 0; i < 62; i++) { + testI64Field(1L << i); + testI64Field(-(1L << i)); + } + + testDouble(); + + testNakedString(""); + testNakedString("short"); + testNakedString("borderlinetiny"); + testNakedString("a bit longer than the smallest possible"); + + testStringField(""); + testStringField("short"); + testStringField("borderlinetiny"); + testStringField("a bit longer than the smallest possible"); + + testNakedBinary(new byte[]{}); + testNakedBinary(new byte[]{0,1,2,3,4,5,6,7,8,9,10}); + testNakedBinary(new byte[]{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14}); + testNakedBinary(new byte[128]); + + testBinaryField(new byte[]{}); + testBinaryField(new byte[]{0,1,2,3,4,5,6,7,8,9,10}); + testBinaryField(new byte[]{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14}); + testBinaryField(new byte[128]); + + testSerialization(OneOfEach.class, Fixtures.oneOfEach); + testSerialization(Nesting.class, Fixtures.nesting); + testSerialization(HolyMoley.class, Fixtures.holyMoley); + testSerialization(CompactProtoTestStruct.class, Fixtures.compactProtoTestStruct); + + testMessage(); + + testServerRequest(); + } + + public static void testNakedByte() throws Exception { + TMemoryBuffer buf = new TMemoryBuffer(0); + TProtocol proto = factory.getProtocol(buf); + proto.writeByte((byte)123); + byte out = proto.readByte(); + if (out != 123) { + throw new RuntimeException("Byte was supposed to be " + (byte)123 + " but was " + out); + } + } + + public static void testByteField(final byte b) throws Exception { + testStructField(new StructFieldTestCase(TType.BYTE, (short)15) { + public void writeMethod(TProtocol proto) throws TException { + proto.writeByte(b); + } + + public void readMethod(TProtocol proto) throws TException { + byte result = proto.readByte(); + if (result != b) { + throw new RuntimeException("Byte was supposed to be " + (byte)b + " but was " + result); + } + } + }); + } + + public static void testNakedI16(short n) throws Exception { + TMemoryBuffer buf = new TMemoryBuffer(0); + TProtocol proto = factory.getProtocol(buf); + proto.writeI16(n); + // System.out.println(buf.inspect()); + int out = proto.readI16(); + if (out != n) { + throw new RuntimeException("I16 was supposed to be " + n + " but was " + out); + } + } + + public static void testI16Field(final short n) throws Exception { + testStructField(new StructFieldTestCase(TType.I16, (short)15) { + public void writeMethod(TProtocol proto) throws TException { + proto.writeI16(n); + } + + public void readMethod(TProtocol proto) throws TException { + short result = proto.readI16(); + if (result != n) { + throw new RuntimeException("I16 was supposed to be " + n + " but was " + result); + } + } + }); + } + + public static void testNakedI32(int n) throws Exception { + TMemoryBuffer buf = new TMemoryBuffer(0); + TProtocol proto = factory.getProtocol(buf); + proto.writeI32(n); + // System.out.println(buf.inspect()); + int out = proto.readI32(); + if (out != n) { + throw new RuntimeException("I32 was supposed to be " + n + " but was " + out); + } + } + + public static void testI32Field(final int n) throws Exception { + testStructField(new StructFieldTestCase(TType.I32, (short)15) { + public void writeMethod(TProtocol proto) throws TException { + proto.writeI32(n); + } + + public void readMethod(TProtocol proto) throws TException { + int result = proto.readI32(); + if (result != n) { + throw new RuntimeException("I32 was supposed to be " + n + " but was " + result); + } + } + }); + + } + + public static void testNakedI64(long n) throws Exception { + TMemoryBuffer buf = new TMemoryBuffer(0); + TProtocol proto = factory.getProtocol(buf); + proto.writeI64(n); + // System.out.println(buf.inspect()); + long out = proto.readI64(); + if (out != n) { + throw new RuntimeException("I64 was supposed to be " + n + " but was " + out); + } + } + + public static void testI64Field(final long n) throws Exception { + testStructField(new StructFieldTestCase(TType.I64, (short)15) { + public void writeMethod(TProtocol proto) throws TException { + proto.writeI64(n); + } + + public void readMethod(TProtocol proto) throws TException { + long result = proto.readI64(); + if (result != n) { + throw new RuntimeException("I64 was supposed to be " + n + " but was " + result); + } + } + }); + } + + public static void testDouble() throws Exception { + TMemoryBuffer buf = new TMemoryBuffer(1000); + TProtocol proto = factory.getProtocol(buf); + proto.writeDouble(123.456); + double out = proto.readDouble(); + if (out != 123.456) { + throw new RuntimeException("Double was supposed to be " + 123.456 + " but was " + out); + } + } + + public static void testNakedString(String str) throws Exception { + TMemoryBuffer buf = new TMemoryBuffer(0); + TProtocol proto = factory.getProtocol(buf); + proto.writeString(str); + // System.out.println(buf.inspect()); + String out = proto.readString(); + if (!str.equals(out)) { + throw new RuntimeException("String was supposed to be '" + str + "' but was '" + out + "'"); + } + } + + public static void testStringField(final String str) throws Exception { + testStructField(new StructFieldTestCase(TType.STRING, (short)15) { + public void writeMethod(TProtocol proto) throws TException { + proto.writeString(str); + } + + public void readMethod(TProtocol proto) throws TException { + String result = proto.readString(); + if (!result.equals(str)) { + throw new RuntimeException("String was supposed to be " + str + " but was " + result); + } + } + }); + } + + public static void testNakedBinary(byte[] data) throws Exception { + TMemoryBuffer buf = new TMemoryBuffer(0); + TProtocol proto = factory.getProtocol(buf); + proto.writeBinary(data); + // System.out.println(buf.inspect()); + byte[] out = proto.readBinary(); + if (!Arrays.equals(data, out)) { + throw new RuntimeException("Binary was supposed to be '" + data + "' but was '" + out + "'"); + } + } + + public static void testBinaryField(final byte[] data) throws Exception { + testStructField(new StructFieldTestCase(TType.STRING, (short)15) { + public void writeMethod(TProtocol proto) throws TException { + proto.writeBinary(data); + } + + public void readMethod(TProtocol proto) throws TException { + byte[] result = proto.readBinary(); + if (!Arrays.equals(data, result)) { + throw new RuntimeException("Binary was supposed to be '" + bytesToString(data) + "' but was '" + bytesToString(result) + "'"); + } + } + }); + + } + + public static void testSerialization(Class klass, T obj) throws Exception { + TMemoryBuffer buf = new TMemoryBuffer(0); + TBinaryProtocol binproto = new TBinaryProtocol(buf); + + try { + obj.write(binproto); + // System.out.println("Size in binary protocol: " + buf.length()); + + buf = new TMemoryBuffer(0); + TProtocol proto = factory.getProtocol(buf); + + obj.write(proto); + System.out.println("Size in compact protocol: " + buf.length()); + // System.out.println(buf.inspect()); + + T objRead = klass.newInstance(); + objRead.read(proto); + if (!obj.equals(objRead)) { + System.out.println("Expected: " + obj.toString()); + System.out.println("Actual: " + objRead.toString()); + // System.out.println(buf.inspect()); + throw new RuntimeException("Objects didn't match!"); + } + } catch (Exception e) { + System.out.println(buf.inspect()); + throw e; + } + } + + public static void testMessage() throws Exception { + List msgs = Arrays.asList(new TMessage[]{ + new TMessage("short message name", TMessageType.CALL, 0), + new TMessage("1", TMessageType.REPLY, 12345), + new TMessage("loooooooooooooooooooooooooooooooooong", TMessageType.EXCEPTION, 1 << 16), + new TMessage("Janky", TMessageType.CALL, 0), + }); + + for (TMessage msg : msgs) { + TMemoryBuffer buf = new TMemoryBuffer(0); + TProtocol proto = factory.getProtocol(buf); + TMessage output = null; + + proto.writeMessageBegin(msg); + proto.writeMessageEnd(); + + output = proto.readMessageBegin(); + + if (!msg.equals(output)) { + throw new RuntimeException("Message was supposed to be " + msg + " but was " + output); + } + } + } + + public static void testServerRequest() throws Exception { + Srv.Iface handler = new Srv.Iface() { + public int Janky(int i32arg) throws TException { + return i32arg * 2; + } + + public int primitiveMethod() throws TException { + // TODO Auto-generated method stub + return 0; + } + + public CompactProtoTestStruct structMethod() throws TException { + // TODO Auto-generated method stub + return null; + } + + public void voidMethod() throws TException { + // TODO Auto-generated method stub + + } + }; + + Srv.Processor testProcessor = new Srv.Processor(handler); + + TMemoryBuffer clientOutTrans = new TMemoryBuffer(0); + TProtocol clientOutProto = factory.getProtocol(clientOutTrans); + TMemoryBuffer clientInTrans = new TMemoryBuffer(0); + TProtocol clientInProto = factory.getProtocol(clientInTrans); + + Srv.Client testClient = new Srv.Client(clientInProto, clientOutProto); + + testClient.send_Janky(1); + // System.out.println(clientOutTrans.inspect()); + testProcessor.process(clientOutProto, clientInProto); + // System.out.println(clientInTrans.inspect()); + int result = testClient.recv_Janky(); + if (result != 2) { + throw new RuntimeException("Got an unexpected result: " + result); + } + } + + // + // Helper methods + // + + private static String bytesToString(byte[] bytes) { + String s = ""; + for (int i = 0; i < bytes.length; i++) { + s += Integer.toHexString((int)bytes[i]) + " "; + } + return s; + } + + private static void testStructField(StructFieldTestCase testCase) throws Exception { + TMemoryBuffer buf = new TMemoryBuffer(0); + TProtocol proto = factory.getProtocol(buf); + + TField field = new TField("test_field", testCase.type_, testCase.id_); + proto.writeStructBegin(new TStruct("test_struct")); + proto.writeFieldBegin(field); + testCase.writeMethod(proto); + proto.writeFieldEnd(); + proto.writeStructEnd(); + + // System.out.println(buf.inspect()); + + proto.readStructBegin(); + TField readField = proto.readFieldBegin(); + // TODO: verify the field is as expected + if (!field.equals(readField)) { + throw new RuntimeException("Expected " + field + " but got " + readField); + } + testCase.readMethod(proto); + proto.readStructEnd(); + } + + public static abstract class StructFieldTestCase { + byte type_; + short id_; + public StructFieldTestCase(byte type, short id) { + type_ = type; + id_ = id; + } + + public abstract void writeMethod(TProtocol proto) throws TException; + public abstract void readMethod(TProtocol proto) throws TException; + } +} \ No newline at end of file diff --git a/lib/java/test/org/apache/thrift/test/TestClient.java b/lib/java/test/org/apache/thrift/test/TestClient.java new file mode 100644 index 00000000..4a95f7a6 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/TestClient.java @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.test; + +// Generated code +import thrift.test.*; + +import org.apache.thrift.TApplicationException; +import org.apache.thrift.TSerializer; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.THttpClient; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TSimpleJSONProtocol; + +import java.util.Map; +import java.util.HashMap; +import java.util.Set; +import java.util.HashSet; +import java.util.List; +import java.util.ArrayList; + +/** + * Test Java client for thrift. Essentially just a copy of the C++ version, + * this makes a variety of requests to enable testing for both performance and + * correctness of the output. + * + */ +public class TestClient { + public static void main(String [] args) { + try { + String host = "localhost"; + int port = 9090; + String url = null; + int numTests = 1; + boolean framed = false; + + int socketTimeout = 1000; + + try { + for (int i = 0; i < args.length; ++i) { + if (args[i].equals("-h")) { + String[] hostport = (args[++i]).split(":"); + host = hostport[0]; + port = Integer.valueOf(hostport[1]); + } else if (args[i].equals("-f") || args[i].equals("-framed")) { + framed = true; + } else if (args[i].equals("-u")) { + url = args[++i]; + } else if (args[i].equals("-n")) { + numTests = Integer.valueOf(args[++i]); + } else if (args[i].equals("-timeout")) { + socketTimeout = Integer.valueOf(args[++i]); + } + } + } catch (Exception x) { + x.printStackTrace(); + } + + TTransport transport; + + if (url != null) { + transport = new THttpClient(url); + } else { + TSocket socket = new TSocket(host, port); + socket.setTimeout(socketTimeout); + transport = socket; + if (framed) { + transport = new TFramedTransport(transport); + } + } + + TBinaryProtocol binaryProtocol = + new TBinaryProtocol(transport); + ThriftTest.Client testClient = + new ThriftTest.Client(binaryProtocol); + Insanity insane = new Insanity(); + + long timeMin = 0; + long timeMax = 0; + long timeTot = 0; + + for (int test = 0; test < numTests; ++test) { + + /** + * CONNECT TEST + */ + System.out.println("Test #" + (test+1) + ", " + "connect " + host + ":" + port); + try { + transport.open(); + } catch (TTransportException ttx) { + System.out.println("Connect failed: " + ttx.getMessage()); + continue; + } + + long start = System.nanoTime(); + + /** + * VOID TEST + */ + try { + System.out.print("testVoid()"); + testClient.testVoid(); + System.out.print(" = void\n"); + } catch (TApplicationException tax) { + tax.printStackTrace(); + } + + /** + * STRING TEST + */ + System.out.print("testString(\"Test\")"); + String s = testClient.testString("Test"); + System.out.print(" = \"" + s + "\"\n"); + + /** + * BYTE TEST + */ + System.out.print("testByte(1)"); + byte i8 = testClient.testByte((byte)1); + System.out.print(" = " + i8 + "\n"); + + /** + * I32 TEST + */ + System.out.print("testI32(-1)"); + int i32 = testClient.testI32(-1); + System.out.print(" = " + i32 + "\n"); + + /** + * I64 TEST + */ + System.out.print("testI64(-34359738368)"); + long i64 = testClient.testI64(-34359738368L); + System.out.print(" = " + i64 + "\n"); + + /** + * DOUBLE TEST + */ + System.out.print("testDouble(5.325098235)"); + double dub = testClient.testDouble(5.325098235); + System.out.print(" = " + dub + "\n"); + + /** + * STRUCT TEST + */ + System.out.print("testStruct({\"Zero\", 1, -3, -5})"); + Xtruct out = new Xtruct(); + out.string_thing = "Zero"; + out.byte_thing = (byte) 1; + out.i32_thing = -3; + out.i64_thing = -5; + Xtruct in = testClient.testStruct(out); + System.out.print(" = {" + "\"" + in.string_thing + "\", " + in.byte_thing + ", " + in.i32_thing + ", " + in.i64_thing + "}\n"); + + /** + * NESTED STRUCT TEST + */ + System.out.print("testNest({1, {\"Zero\", 1, -3, -5}), 5}"); + Xtruct2 out2 = new Xtruct2(); + out2.byte_thing = (short)1; + out2.struct_thing = out; + out2.i32_thing = 5; + Xtruct2 in2 = testClient.testNest(out2); + in = in2.struct_thing; + System.out.print(" = {" + in2.byte_thing + ", {" + "\"" + in.string_thing + "\", " + in.byte_thing + ", " + in.i32_thing + ", " + in.i64_thing + "}, " + in2.i32_thing + "}\n"); + + /** + * MAP TEST + */ + Map mapout = new HashMap(); + for (int i = 0; i < 5; ++i) { + mapout.put(i, i-10); + } + System.out.print("testMap({"); + boolean first = true; + for (int key : mapout.keySet()) { + if (first) { + first = false; + } else { + System.out.print(", "); + } + System.out.print(key + " => " + mapout.get(key)); + } + System.out.print("})"); + Map mapin = testClient.testMap(mapout); + System.out.print(" = {"); + first = true; + for (int key : mapin.keySet()) { + if (first) { + first = false; + } else { + System.out.print(", "); + } + System.out.print(key + " => " + mapout.get(key)); + } + System.out.print("}\n"); + + /** + * SET TEST + */ + Set setout = new HashSet(); + for (int i = -2; i < 3; ++i) { + setout.add(i); + } + System.out.print("testSet({"); + first = true; + for (int elem : setout) { + if (first) { + first = false; + } else { + System.out.print(", "); + } + System.out.print(elem); + } + System.out.print("})"); + Set setin = testClient.testSet(setout); + System.out.print(" = {"); + first = true; + for (int elem : setin) { + if (first) { + first = false; + } else { + System.out.print(", "); + } + System.out.print(elem); + } + System.out.print("}\n"); + + /** + * LIST TEST + */ + List listout = new ArrayList(); + for (int i = -2; i < 3; ++i) { + listout.add(i); + } + System.out.print("testList({"); + first = true; + for (int elem : listout) { + if (first) { + first = false; + } else { + System.out.print(", "); + } + System.out.print(elem); + } + System.out.print("})"); + List listin = testClient.testList(listout); + System.out.print(" = {"); + first = true; + for (int elem : listin) { + if (first) { + first = false; + } else { + System.out.print(", "); + } + System.out.print(elem); + } + System.out.print("}\n"); + + /** + * ENUM TEST + */ + System.out.print("testEnum(ONE)"); + int ret = testClient.testEnum(Numberz.ONE); + System.out.print(" = " + ret + "\n"); + + System.out.print("testEnum(TWO)"); + ret = testClient.testEnum(Numberz.TWO); + System.out.print(" = " + ret + "\n"); + + System.out.print("testEnum(THREE)"); + ret = testClient.testEnum(Numberz.THREE); + System.out.print(" = " + ret + "\n"); + + System.out.print("testEnum(FIVE)"); + ret = testClient.testEnum(Numberz.FIVE); + System.out.print(" = " + ret + "\n"); + + System.out.print("testEnum(EIGHT)"); + ret = testClient.testEnum(Numberz.EIGHT); + System.out.print(" = " + ret + "\n"); + + /** + * TYPEDEF TEST + */ + System.out.print("testTypedef(309858235082523)"); + long uid = testClient.testTypedef(309858235082523L); + System.out.print(" = " + uid + "\n"); + + /** + * NESTED MAP TEST + */ + System.out.print("testMapMap(1)"); + Map> mm = + testClient.testMapMap(1); + System.out.print(" = {"); + for (int key : mm.keySet()) { + System.out.print(key + " => {"); + Map m2 = mm.get(key); + for (int k2 : m2.keySet()) { + System.out.print(k2 + " => " + m2.get(k2) + ", "); + } + System.out.print("}, "); + } + System.out.print("}\n"); + + /** + * INSANITY TEST + */ + insane = new Insanity(); + insane.userMap = new HashMap(); + insane.userMap.put(Numberz.FIVE, (long)5000); + Xtruct truck = new Xtruct(); + truck.string_thing = "Truck"; + truck.byte_thing = (byte)8; + truck.i32_thing = 8; + truck.i64_thing = 8; + insane.xtructs = new ArrayList(); + insane.xtructs.add(truck); + System.out.print("testInsanity()"); + Map> whoa = + testClient.testInsanity(insane); + System.out.print(" = {"); + for (long key : whoa.keySet()) { + Map val = whoa.get(key); + System.out.print(key + " => {"); + + for (int k2 : val.keySet()) { + Insanity v2 = val.get(k2); + System.out.print(k2 + " => {"); + Map userMap = v2.userMap; + System.out.print("{"); + if (userMap != null) { + for (int k3 : userMap.keySet()) { + System.out.print(k3 + " => " + userMap.get(k3) + ", "); + } + } + System.out.print("}, "); + + List xtructs = v2.xtructs; + System.out.print("{"); + if (xtructs != null) { + for (Xtruct x : xtructs) { + System.out.print("{" + "\"" + x.string_thing + "\", " + x.byte_thing + ", " + x.i32_thing + ", "+ x.i64_thing + "}, "); + } + } + System.out.print("}"); + + System.out.print("}, "); + } + System.out.print("}, "); + } + System.out.print("}\n"); + + // Test oneway + System.out.print("testOneway(3)..."); + long startOneway = System.nanoTime(); + testClient.testOneway(3); + long onewayElapsedMillis = (System.nanoTime() - startOneway) / 1000000; + if (onewayElapsedMillis > 200) { + throw new Exception("Oneway test failed: took " + + Long.toString(onewayElapsedMillis) + + "ms"); + } else { + System.out.println("Success - took " + + Long.toString(onewayElapsedMillis) + + "ms"); + } + + + long stop = System.nanoTime(); + long tot = stop-start; + + System.out.println("Total time: " + tot/1000 + "us"); + + if (timeMin == 0 || tot < timeMin) { + timeMin = tot; + } + if (tot > timeMax) { + timeMax = tot; + } + timeTot += tot; + + transport.close(); + } + + long timeAvg = timeTot / numTests; + + System.out.println("Min time: " + timeMin/1000 + "us"); + System.out.println("Max time: " + timeMax/1000 + "us"); + System.out.println("Avg time: " + timeAvg/1000 + "us"); + + String json = (new TSerializer(new TSimpleJSONProtocol.Factory())).toString(insane); + + System.out.println("\nFor good meausre here is some JSON:\n" + json); + + } catch (Exception x) { + x.printStackTrace(); + } + + } + +} diff --git a/lib/java/test/org/apache/thrift/test/TestNonblockingServer.java b/lib/java/test/org/apache/thrift/test/TestNonblockingServer.java new file mode 100644 index 00000000..2e6a1780 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/TestNonblockingServer.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.test; + +import org.apache.thrift.server.THsHaServer; +import org.apache.thrift.server.TNonblockingServer; +import org.apache.thrift.server.TServer; +import org.apache.thrift.transport.TNonblockingServerSocket; + +import thrift.test.ThriftTest; + + +public class TestNonblockingServer extends TestServer { + public static void main(String [] args) { + try { + int port = 9090; + boolean hsha = false; + + for (int i = 0; i < args.length; i++) { + if (args[i].equals("-p")) { + port = Integer.valueOf(args[i++]); + } else if (args[i].equals("-hsha")) { + hsha = true; + } + } + + // Processor + TestHandler testHandler = + new TestHandler(); + ThriftTest.Processor testProcessor = + new ThriftTest.Processor(testHandler); + + // Transport + TNonblockingServerSocket tServerSocket = + new TNonblockingServerSocket(port); + + TServer serverEngine; + + if (hsha) { + // HsHa Server + serverEngine = new THsHaServer(testProcessor, tServerSocket); + } else { + // Nonblocking Server + serverEngine = new TNonblockingServer(testProcessor, tServerSocket); + } + + // Run it + System.out.println("Starting the server on port " + port + "..."); + serverEngine.serve(); + + } catch (Exception x) { + x.printStackTrace(); + } + System.out.println("done."); + } +} diff --git a/lib/java/test/org/apache/thrift/test/TestServer.java b/lib/java/test/org/apache/thrift/test/TestServer.java new file mode 100644 index 00000000..986f8890 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/TestServer.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.server.TServer; +import org.apache.thrift.server.TThreadPoolServer; +import org.apache.thrift.transport.TServerSocket; + +import thrift.test.Insanity; +import thrift.test.Numberz; +import thrift.test.ThriftTest; +import thrift.test.Xception; +import thrift.test.Xception2; +import thrift.test.Xtruct; +import thrift.test.Xtruct2; + +public class TestServer { + + public static class TestHandler implements ThriftTest.Iface { + + public TestHandler() {} + + public void testVoid() { + System.out.print("testVoid()\n"); + } + + public String testString(String thing) { + System.out.print("testString(\"" + thing + "\")\n"); + return thing; + } + + public byte testByte(byte thing) { + System.out.print("testByte(" + thing + ")\n"); + return thing; + } + + public int testI32(int thing) { + System.out.print("testI32(" + thing + ")\n"); + return thing; + } + + public long testI64(long thing) { + System.out.print("testI64(" + thing + ")\n"); + return thing; + } + + public double testDouble(double thing) { + System.out.print("testDouble(" + thing + ")\n"); + return thing; + } + + public Xtruct testStruct(Xtruct thing) { + System.out.print("testStruct({" + + "\"" + thing.string_thing + "\", " + + thing.byte_thing + ", " + + thing.i32_thing + ", " + + thing.i64_thing + "})\n"); + return thing; + } + + public Xtruct2 testNest(Xtruct2 nest) { + Xtruct thing = nest.struct_thing; + System.out.print("testNest({" + + nest.byte_thing + ", {" + + "\"" + thing.string_thing + "\", " + + thing.byte_thing + ", " + + thing.i32_thing + ", " + + thing.i64_thing + "}, " + + nest.i32_thing + "})\n"); + return nest; + } + + public Map testMap(Map thing) { + System.out.print("testMap({"); + boolean first = true; + for (int key : thing.keySet()) { + if (first) { + first = false; + } else { + System.out.print(", "); + } + System.out.print(key + " => " + thing.get(key)); + } + System.out.print("})\n"); + return thing; + } + + public Set testSet(Set thing) { + System.out.print("testSet({"); + boolean first = true; + for (int elem : thing) { + if (first) { + first = false; + } else { + System.out.print(", "); + } + System.out.print(elem); + } + System.out.print("})\n"); + return thing; + } + + public List testList(List thing) { + System.out.print("testList({"); + boolean first = true; + for (int elem : thing) { + if (first) { + first = false; + } else { + System.out.print(", "); + } + System.out.print(elem); + } + System.out.print("})\n"); + return thing; + } + + public int testEnum(int thing) { + System.out.print("testEnum(" + thing + ")\n"); + return thing; + } + + public long testTypedef(long thing) { + System.out.print("testTypedef(" + thing + ")\n"); + return thing; + } + + public Map> testMapMap(int hello) { + System.out.print("testMapMap(" + hello + ")\n"); + Map> mapmap = + new HashMap>(); + + HashMap pos = new HashMap(); + HashMap neg = new HashMap(); + for (int i = 1; i < 5; i++) { + pos.put(i, i); + neg.put(-i, -i); + } + + mapmap.put(4, pos); + mapmap.put(-4, neg); + + return mapmap; + } + + public Map> testInsanity(Insanity argument) { + System.out.print("testInsanity()\n"); + + Xtruct hello = new Xtruct(); + hello.string_thing = "Hello2"; + hello.byte_thing = 2; + hello.i32_thing = 2; + hello.i64_thing = 2; + + Xtruct goodbye = new Xtruct(); + goodbye.string_thing = "Goodbye4"; + goodbye.byte_thing = (byte)4; + goodbye.i32_thing = 4; + goodbye.i64_thing = (long)4; + + Insanity crazy = new Insanity(); + crazy.userMap = new HashMap(); + crazy.xtructs = new ArrayList(); + + crazy.userMap.put(Numberz.EIGHT, (long)8); + crazy.xtructs.add(goodbye); + + Insanity looney = new Insanity(); + crazy.userMap.put(Numberz.FIVE, (long)5); + crazy.xtructs.add(hello); + + HashMap first_map = new HashMap(); + HashMap second_map = new HashMap();; + + first_map.put(Numberz.TWO, crazy); + first_map.put(Numberz.THREE, crazy); + + second_map.put(Numberz.SIX, looney); + + Map> insane = + new HashMap>(); + insane.put((long)1, first_map); + insane.put((long)2, second_map); + + return insane; + } + + public Xtruct testMulti(byte arg0, int arg1, long arg2, Map arg3, int arg4, long arg5) { + System.out.print("testMulti()\n"); + + Xtruct hello = new Xtruct();; + hello.string_thing = "Hello2"; + hello.byte_thing = arg0; + hello.i32_thing = arg1; + hello.i64_thing = arg2; + return hello; + } + + public void testException(String arg) throws Xception { + System.out.print("testException("+arg+")\n"); + if (arg.equals("Xception")) { + Xception x = new Xception(); + x.errorCode = 1001; + x.message = "This is an Xception"; + throw x; + } + return; + } + + public Xtruct testMultiException(String arg0, String arg1) throws Xception, Xception2 { + System.out.print("testMultiException(" + arg0 + ", " + arg1 + ")\n"); + if (arg0.equals("Xception")) { + Xception x = new Xception(); + x.errorCode = 1001; + x.message = "This is an Xception"; + throw x; + } else if (arg0.equals("Xception2")) { + Xception2 x = new Xception2(); + x.errorCode = 2002; + x.struct_thing = new Xtruct(); + x.struct_thing.string_thing = "This is an Xception2"; + throw x; + } + + Xtruct result = new Xtruct(); + result.string_thing = arg1; + return result; + } + + public void testOneway(int sleepFor) { + System.out.println("testOneway(" + Integer.toString(sleepFor) + + ") => sleeping..."); + try { + Thread.sleep(sleepFor * 1000); + System.out.println("Done sleeping!"); + } catch (InterruptedException ie) { + throw new RuntimeException(ie); + } + } + + } // class TestHandler + + public static void main(String [] args) { + try { + int port = 9090; + if (args.length > 1) { + port = Integer.valueOf(args[0]); + } + + // Processor + TestHandler testHandler = + new TestHandler(); + ThriftTest.Processor testProcessor = + new ThriftTest.Processor(testHandler); + + // Transport + TServerSocket tServerSocket = + new TServerSocket(port); + + // Protocol factory + TProtocolFactory tProtocolFactory = + new TBinaryProtocol.Factory(); + + TServer serverEngine; + + // Simple Server + // serverEngine = new TSimpleServer(testProcessor, tServerSocket); + + // ThreadPool Server + serverEngine = new TThreadPoolServer(testProcessor, tServerSocket, tProtocolFactory); + + // Run it + System.out.println("Starting the server on port " + port + "..."); + serverEngine.serve(); + + } catch (Exception x) { + x.printStackTrace(); + } + System.out.println("done."); + } +} diff --git a/lib/java/test/org/apache/thrift/test/ToStringTest.java b/lib/java/test/org/apache/thrift/test/ToStringTest.java new file mode 100644 index 00000000..569a61c4 --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/ToStringTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.test; + +import thrift.test.*; + +/** + */ +public class ToStringTest { + public static void main(String[] args) throws Exception { + JavaTestHelper object = new JavaTestHelper(); + object.req_int = 0; + object.req_obj = ""; + + + object.req_bin = new byte[] { + 0, -1, 2, -3, 4, -5, 6, -7, 8, -9, 10, -11, 12, -13, 14, -15, + 16, -17, 18, -19, 20, -21, 22, -23, 24, -25, 26, -27, 28, -29, + 30, -31, 32, -33, 34, -35, 36, -37, 38, -39, 40, -41, 42, -43, 44, + -45, 46, -47, 48, -49, 50, -51, 52, -53, 54, -55, 56, -57, 58, -59, + 60, -61, 62, -63, 64, -65, 66, -67, 68, -69, 70, -71, 72, -73, 74, + -75, 76, -77, 78, -79, 80, -81, 82, -83, 84, -85, 86, -87, 88, -89, + 90, -91, 92, -93, 94, -95, 96, -97, 98, -99, 100, -101, 102, -103, + 104, -105, 106, -107, 108, -109, 110, -111, 112, -113, 114, -115, + 116, -117, 118, -119, 120, -121, 122, -123, 124, -125, 126, -127, + }; + + if (!object.toString().equals( + "JavaTestHelper(req_int:0, req_obj:, req_bin:"+ + "00 FF 02 FD 04 FB 06 F9 08 F7 0A F5 0C F3 0E F1 10 EF 12 ED 14 "+ + "EB 16 E9 18 E7 1A E5 1C E3 1E E1 20 DF 22 DD 24 DB 26 D9 28 D7 "+ + "2A D5 2C D3 2E D1 30 CF 32 CD 34 CB 36 C9 38 C7 3A C5 3C C3 3E "+ + "C1 40 BF 42 BD 44 BB 46 B9 48 B7 4A B5 4C B3 4E B1 50 AF 52 AD "+ + "54 AB 56 A9 58 A7 5A A5 5C A3 5E A1 60 9F 62 9D 64 9B 66 99 68 "+ + "97 6A 95 6C 93 6E 91 70 8F 72 8D 74 8B 76 89 78 87 7A 85 7C 83 "+ + "7E 81)")) { + throw new RuntimeException(); + } + + object.req_bin = new byte[] { + 0, -1, 2, -3, 4, -5, 6, -7, 8, -9, 10, -11, 12, -13, 14, -15, + 16, -17, 18, -19, 20, -21, 22, -23, 24, -25, 26, -27, 28, -29, + 30, -31, 32, -33, 34, -35, 36, -37, 38, -39, 40, -41, 42, -43, 44, + -45, 46, -47, 48, -49, 50, -51, 52, -53, 54, -55, 56, -57, 58, -59, + 60, -61, 62, -63, 64, -65, 66, -67, 68, -69, 70, -71, 72, -73, 74, + -75, 76, -77, 78, -79, 80, -81, 82, -83, 84, -85, 86, -87, 88, -89, + 90, -91, 92, -93, 94, -95, 96, -97, 98, -99, 100, -101, 102, -103, + 104, -105, 106, -107, 108, -109, 110, -111, 112, -113, 114, -115, + 116, -117, 118, -119, 120, -121, 122, -123, 124, -125, 126, -127, + 0, + }; + + if (!object.toString().equals( + "JavaTestHelper(req_int:0, req_obj:, req_bin:"+ + "00 FF 02 FD 04 FB 06 F9 08 F7 0A F5 0C F3 0E F1 10 EF 12 ED 14 "+ + "EB 16 E9 18 E7 1A E5 1C E3 1E E1 20 DF 22 DD 24 DB 26 D9 28 D7 "+ + "2A D5 2C D3 2E D1 30 CF 32 CD 34 CB 36 C9 38 C7 3A C5 3C C3 3E "+ + "C1 40 BF 42 BD 44 BB 46 B9 48 B7 4A B5 4C B3 4E B1 50 AF 52 AD "+ + "54 AB 56 A9 58 A7 5A A5 5C A3 5E A1 60 9F 62 9D 64 9B 66 99 68 "+ + "97 6A 95 6C 93 6E 91 70 8F 72 8D 74 8B 76 89 78 87 7A 85 7C 83 "+ + "7E 81 ...)")) { + throw new RuntimeException(); + } + + object.req_bin = new byte[] {}; + object.setOpt_binIsSet(true); + + + if (!object.toString().equals( + "JavaTestHelper(req_int:0, req_obj:, req_bin:)")) { + throw new RuntimeException(); + } + } +} + diff --git a/lib/java/test/org/apache/thrift/test/WriteStruct.java b/lib/java/test/org/apache/thrift/test/WriteStruct.java new file mode 100644 index 00000000..474c808e --- /dev/null +++ b/lib/java/test/org/apache/thrift/test/WriteStruct.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.test; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; + +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.thrift.transport.TTransport; + +public class WriteStruct { + public static void main(String[] args) throws Exception { + if (args.length != 2) { + System.out.println("usage: java -cp build/classes org.apache.thrift.test.WriteStruct filename proto_factory_class"); + System.out.println("Write out an instance of Fixtures.compactProtocolTestStruct to 'file'. Use a protocol from 'proto_factory_class'."); + } + + TTransport trans = new TIOStreamTransport(new BufferedOutputStream(new FileOutputStream(args[0]))); + + TProtocolFactory factory = (TProtocolFactory)Class.forName(args[1]).newInstance(); + + TProtocol proto = factory.getProtocol(trans); + + Fixtures.compactProtoTestStruct.write(proto); + trans.flush(); + } + +} diff --git a/lib/ocaml/Makefile b/lib/ocaml/Makefile new file mode 100644 index 00000000..6abeee71 --- /dev/null +++ b/lib/ocaml/Makefile @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +all: + cd src; make; cd .. +clean: + cd src; make clean; cd .. diff --git a/lib/ocaml/OCamlMakefile b/lib/ocaml/OCamlMakefile new file mode 100644 index 00000000..b0b9252c --- /dev/null +++ b/lib/ocaml/OCamlMakefile @@ -0,0 +1,1231 @@ +########################################################################### +# OCamlMakefile +# Copyright (C) 1999-2007 Markus Mottl +# +# For updates see: +# http://www.ocaml.info/home/ocaml_sources.html +# +########################################################################### + +# Modified by damien for .glade.ml compilation + +# Set these variables to the names of the sources to be processed and +# the result variable. Order matters during linkage! + +ifndef SOURCES + SOURCES := foo.ml +endif +export SOURCES + +ifndef RES_CLIB_SUF + RES_CLIB_SUF := _stubs +endif +export RES_CLIB_SUF + +ifndef RESULT + RESULT := foo +endif +export RESULT := $(strip $(RESULT)) + +export LIB_PACK_NAME + +ifndef DOC_FILES + DOC_FILES := $(filter %.mli, $(SOURCES)) +endif +export DOC_FILES +FIRST_DOC_FILE := $(firstword $(DOC_FILES)) + +export BCSUFFIX +export NCSUFFIX + +ifndef TOPSUFFIX + TOPSUFFIX := .top +endif +export TOPSUFFIX + +# Eventually set include- and library-paths, libraries to link, +# additional compilation-, link- and ocamlyacc-flags +# Path- and library information needs not be written with "-I" and such... +# Define THREADS if you need it, otherwise leave it unset (same for +# USE_CAMLP4)! + +export THREADS +export VMTHREADS +export ANNOTATE +export USE_CAMLP4 + +export INCDIRS +export LIBDIRS +export EXTLIBDIRS +export RESULTDEPS +export OCAML_DEFAULT_DIRS + +export LIBS +export CLIBS +export CFRAMEWORKS + +export OCAMLFLAGS +export OCAMLNCFLAGS +export OCAMLBCFLAGS + +export OCAMLLDFLAGS +export OCAMLNLDFLAGS +export OCAMLBLDFLAGS + +export OCAMLMKLIB_FLAGS + +ifndef OCAMLCPFLAGS + OCAMLCPFLAGS := a +endif +export OCAMLCPFLAGS + +ifndef DOC_DIR + DOC_DIR := doc +endif +export DOC_DIR + +export PPFLAGS + +export LFLAGS +export YFLAGS +export IDLFLAGS + +export OCAMLDOCFLAGS + +export OCAMLFIND_INSTFLAGS + +export DVIPSFLAGS + +export STATIC + +# Add a list of optional trash files that should be deleted by "make clean" +export TRASH + +ECHO := echo + +ifdef REALLY_QUIET + export REALLY_QUIET + ECHO := true + LFLAGS := $(LFLAGS) -q + YFLAGS := $(YFLAGS) -q +endif + +#################### variables depending on your OCaml-installation + +ifdef MINGW + export MINGW + WIN32 := 1 + CFLAGS_WIN32 := -mno-cygwin +endif +ifdef MSVC + export MSVC + WIN32 := 1 + ifndef STATIC + CPPFLAGS_WIN32 := -DCAML_DLL + endif + CFLAGS_WIN32 += -nologo + EXT_OBJ := obj + EXT_LIB := lib + ifeq ($(CC),gcc) + # work around GNU Make default value + ifdef THREADS + CC := cl -MT + else + CC := cl + endif + endif + ifeq ($(CXX),g++) + # work around GNU Make default value + CXX := $(CC) + endif + CFLAG_O := -Fo +endif +ifdef WIN32 + EXT_CXX := cpp + EXE := .exe +endif + +ifndef EXT_OBJ + EXT_OBJ := o +endif +ifndef EXT_LIB + EXT_LIB := a +endif +ifndef EXT_CXX + EXT_CXX := cc +endif +ifndef EXE + EXE := # empty +endif +ifndef CFLAG_O + CFLAG_O := -o # do not delete this comment (preserves trailing whitespace)! +endif + +export CC +export CXX +export CFLAGS +export CXXFLAGS +export LDFLAGS +export CPPFLAGS + +ifndef RPATH_FLAG + ifdef ELF_RPATH_FLAG + RPATH_FLAG := $(ELF_RPATH_FLAG) + else + RPATH_FLAG := -R + endif +endif +export RPATH_FLAG + +ifndef MSVC +ifndef PIC_CFLAGS + PIC_CFLAGS := -fPIC +endif +ifndef PIC_CPPFLAGS + PIC_CPPFLAGS := -DPIC +endif +endif + +export PIC_CFLAGS +export PIC_CPPFLAGS + +BCRESULT := $(addsuffix $(BCSUFFIX), $(RESULT)) +NCRESULT := $(addsuffix $(NCSUFFIX), $(RESULT)) +TOPRESULT := $(addsuffix $(TOPSUFFIX), $(RESULT)) + +ifndef OCAMLFIND + OCAMLFIND := ocamlfind +endif +export OCAMLFIND + +ifndef OCAMLC + OCAMLC := ocamlc +endif +export OCAMLC + +ifndef OCAMLOPT + OCAMLOPT := ocamlopt +endif +export OCAMLOPT + +ifndef OCAMLMKTOP + OCAMLMKTOP := ocamlmktop +endif +export OCAMLMKTOP + +ifndef OCAMLCP + OCAMLCP := ocamlcp +endif +export OCAMLCP + +ifndef OCAMLDEP + OCAMLDEP := ocamldep +endif +export OCAMLDEP + +ifndef OCAMLLEX + OCAMLLEX := ocamllex +endif +export OCAMLLEX + +ifndef OCAMLYACC + OCAMLYACC := ocamlyacc +endif +export OCAMLYACC + +ifndef OCAMLMKLIB + OCAMLMKLIB := ocamlmklib +endif +export OCAMLMKLIB + +ifndef OCAML_GLADECC + OCAML_GLADECC := lablgladecc2 +endif +export OCAML_GLADECC + +ifndef OCAML_GLADECC_FLAGS + OCAML_GLADECC_FLAGS := +endif +export OCAML_GLADECC_FLAGS + +ifndef CAMELEON_REPORT + CAMELEON_REPORT := report +endif +export CAMELEON_REPORT + +ifndef CAMELEON_REPORT_FLAGS + CAMELEON_REPORT_FLAGS := +endif +export CAMELEON_REPORT_FLAGS + +ifndef CAMELEON_ZOGGY + CAMELEON_ZOGGY := camlp4o pa_zog.cma pr_o.cmo +endif +export CAMELEON_ZOGGY + +ifndef CAMELEON_ZOGGY_FLAGS + CAMELEON_ZOGGY_FLAGS := +endif +export CAMELEON_ZOGGY_FLAGS + +ifndef OXRIDL + OXRIDL := oxridl +endif +export OXRIDL + +ifndef CAMLIDL + CAMLIDL := camlidl +endif +export CAMLIDL + +ifndef CAMLIDLDLL + CAMLIDLDLL := camlidldll +endif +export CAMLIDLDLL + +ifndef NOIDLHEADER + MAYBE_IDL_HEADER := -header +endif +export NOIDLHEADER + +export NO_CUSTOM + +ifndef CAMLP4 + CAMLP4 := camlp4 +endif +export CAMLP4 + +ifndef REAL_OCAMLFIND + ifdef PACKS + ifndef CREATE_LIB + ifdef THREADS + PACKS += threads + endif + endif + empty := + space := $(empty) $(empty) + comma := , + ifdef PREDS + PRE_OCAML_FIND_PREDICATES := $(subst $(space),$(comma),$(PREDS)) + PRE_OCAML_FIND_PACKAGES := $(subst $(space),$(comma),$(PACKS)) + OCAML_FIND_PREDICATES := -predicates $(PRE_OCAML_FIND_PREDICATES) + # OCAML_DEP_PREDICATES := -syntax $(PRE_OCAML_FIND_PREDICATES) + OCAML_FIND_PACKAGES := $(OCAML_FIND_PREDICATES) -package $(PRE_OCAML_FIND_PACKAGES) + OCAML_DEP_PACKAGES := $(OCAML_DEP_PREDICATES) -package $(PRE_OCAML_FIND_PACKAGES) + else + OCAML_FIND_PACKAGES := -package $(subst $(space),$(comma),$(PACKS)) + OCAML_DEP_PACKAGES := + endif + OCAML_FIND_LINKPKG := -linkpkg + REAL_OCAMLFIND := $(OCAMLFIND) + endif +endif + +export OCAML_FIND_PACKAGES +export OCAML_DEP_PACKAGES +export OCAML_FIND_LINKPKG +export REAL_OCAMLFIND + +ifndef OCAMLDOC + OCAMLDOC := ocamldoc +endif +export OCAMLDOC + +ifndef LATEX + LATEX := latex +endif +export LATEX + +ifndef DVIPS + DVIPS := dvips +endif +export DVIPS + +ifndef PS2PDF + PS2PDF := ps2pdf +endif +export PS2PDF + +ifndef OCAMLMAKEFILE + OCAMLMAKEFILE := OCamlMakefile +endif +export OCAMLMAKEFILE + +ifndef OCAMLLIBPATH + OCAMLLIBPATH := \ + $(shell $(OCAMLC) 2>/dev/null -where || echo /usr/local/lib/ocaml) +endif +export OCAMLLIBPATH + +ifndef OCAML_LIB_INSTALL + OCAML_LIB_INSTALL := $(OCAMLLIBPATH)/contrib +endif +export OCAML_LIB_INSTALL + +########################################################################### + +#################### change following sections only if +#################### you know what you are doing! + +# delete target files when a build command fails +.PHONY: .DELETE_ON_ERROR +.DELETE_ON_ERROR: + +# for pedants using "--warn-undefined-variables" +export MAYBE_IDL +export REAL_RESULT +export CAMLIDLFLAGS +export THREAD_FLAG +export RES_CLIB +export MAKEDLL +export ANNOT_FLAG +export C_OXRIDL +export SUBPROJS +export CFLAGS_WIN32 +export CPPFLAGS_WIN32 + +INCFLAGS := + +SHELL := /bin/sh + +MLDEPDIR := ._d +BCDIDIR := ._bcdi +NCDIDIR := ._ncdi + +FILTER_EXTNS := %.mli %.ml %.mll %.mly %.idl %.oxridl %.c %.m %.$(EXT_CXX) %.rep %.zog %.glade + +FILTERED := $(filter $(FILTER_EXTNS), $(SOURCES)) +SOURCE_DIRS := $(filter-out ./, $(sort $(dir $(FILTERED)))) + +FILTERED_REP := $(filter %.rep, $(FILTERED)) +DEP_REP := $(FILTERED_REP:%.rep=$(MLDEPDIR)/%.d) +AUTO_REP := $(FILTERED_REP:.rep=.ml) + +FILTERED_ZOG := $(filter %.zog, $(FILTERED)) +DEP_ZOG := $(FILTERED_ZOG:%.zog=$(MLDEPDIR)/%.d) +AUTO_ZOG := $(FILTERED_ZOG:.zog=.ml) + +FILTERED_GLADE := $(filter %.glade, $(FILTERED)) +DEP_GLADE := $(FILTERED_GLADE:%.glade=$(MLDEPDIR)/%.d) +AUTO_GLADE := $(FILTERED_GLADE:.glade=.ml) + +FILTERED_ML := $(filter %.ml, $(FILTERED)) +DEP_ML := $(FILTERED_ML:%.ml=$(MLDEPDIR)/%.d) + +FILTERED_MLI := $(filter %.mli, $(FILTERED)) +DEP_MLI := $(FILTERED_MLI:.mli=.di) + +FILTERED_MLL := $(filter %.mll, $(FILTERED)) +DEP_MLL := $(FILTERED_MLL:%.mll=$(MLDEPDIR)/%.d) +AUTO_MLL := $(FILTERED_MLL:.mll=.ml) + +FILTERED_MLY := $(filter %.mly, $(FILTERED)) +DEP_MLY := $(FILTERED_MLY:%.mly=$(MLDEPDIR)/%.d) $(FILTERED_MLY:.mly=.di) +AUTO_MLY := $(FILTERED_MLY:.mly=.mli) $(FILTERED_MLY:.mly=.ml) + +FILTERED_IDL := $(filter %.idl, $(FILTERED)) +DEP_IDL := $(FILTERED_IDL:%.idl=$(MLDEPDIR)/%.d) $(FILTERED_IDL:.idl=.di) +C_IDL := $(FILTERED_IDL:%.idl=%_stubs.c) +ifndef NOIDLHEADER + C_IDL += $(FILTERED_IDL:.idl=.h) +endif +OBJ_C_IDL := $(FILTERED_IDL:%.idl=%_stubs.$(EXT_OBJ)) +AUTO_IDL := $(FILTERED_IDL:.idl=.mli) $(FILTERED_IDL:.idl=.ml) $(C_IDL) + +FILTERED_OXRIDL := $(filter %.oxridl, $(FILTERED)) +DEP_OXRIDL := $(FILTERED_OXRIDL:%.oxridl=$(MLDEPDIR)/%.d) $(FILTERED_OXRIDL:.oxridl=.di) +AUTO_OXRIDL := $(FILTERED_OXRIDL:.oxridl=.mli) $(FILTERED_OXRIDL:.oxridl=.ml) $(C_OXRIDL) + +FILTERED_C_CXX := $(filter %.c %.m %.$(EXT_CXX), $(FILTERED)) +OBJ_C_CXX := $(FILTERED_C_CXX:.c=.$(EXT_OBJ)) +OBJ_C_CXX := $(OBJ_C_CXX:.m=.$(EXT_OBJ)) +OBJ_C_CXX := $(OBJ_C_CXX:.$(EXT_CXX)=.$(EXT_OBJ)) + +PRE_TARGETS += $(AUTO_MLL) $(AUTO_MLY) $(AUTO_IDL) $(AUTO_OXRIDL) $(AUTO_ZOG) $(AUTO_REP) $(AUTO_GLADE) + +ALL_DEPS := $(DEP_ML) $(DEP_MLI) $(DEP_MLL) $(DEP_MLY) $(DEP_IDL) $(DEP_OXRIDL) $(DEP_ZOG) $(DEP_REP) $(DEP_GLADE) + +MLDEPS := $(filter %.d, $(ALL_DEPS)) +MLIDEPS := $(filter %.di, $(ALL_DEPS)) +BCDEPIS := $(MLIDEPS:%.di=$(BCDIDIR)/%.di) +NCDEPIS := $(MLIDEPS:%.di=$(NCDIDIR)/%.di) + +ALLML := $(filter %.mli %.ml %.mll %.mly %.idl %.oxridl %.rep %.zog %.glade, $(FILTERED)) + +IMPLO_INTF := $(ALLML:%.mli=%.mli.__) +IMPLO_INTF := $(foreach file, $(IMPLO_INTF), \ + $(basename $(file)).cmi $(basename $(file)).cmo) +IMPLO_INTF := $(filter-out %.mli.cmo, $(IMPLO_INTF)) +IMPLO_INTF := $(IMPLO_INTF:%.mli.cmi=%.cmi) + +IMPLX_INTF := $(IMPLO_INTF:.cmo=.cmx) + +INTF := $(filter %.cmi, $(IMPLO_INTF)) +IMPL_CMO := $(filter %.cmo, $(IMPLO_INTF)) +IMPL_CMX := $(IMPL_CMO:.cmo=.cmx) +IMPL_ASM := $(IMPL_CMO:.cmo=.asm) +IMPL_S := $(IMPL_CMO:.cmo=.s) + +OBJ_LINK := $(OBJ_C_IDL) $(OBJ_C_CXX) +OBJ_FILES := $(IMPL_CMO:.cmo=.$(EXT_OBJ)) $(OBJ_LINK) + +EXECS := $(addsuffix $(EXE), \ + $(sort $(TOPRESULT) $(BCRESULT) $(NCRESULT))) +ifdef WIN32 + EXECS += $(BCRESULT).dll $(NCRESULT).dll +endif + +CLIB_BASE := $(RESULT)$(RES_CLIB_SUF) +ifneq ($(strip $(OBJ_LINK)),) + RES_CLIB := lib$(CLIB_BASE).$(EXT_LIB) +endif + +ifdef WIN32 +DLLSONAME := $(CLIB_BASE).dll +else +DLLSONAME := dll$(CLIB_BASE).so +endif + +NONEXECS := $(INTF) $(IMPL_CMO) $(IMPL_CMX) $(IMPL_ASM) $(IMPL_S) \ + $(OBJ_FILES) $(PRE_TARGETS) $(BCRESULT).cma $(NCRESULT).cmxa \ + $(NCRESULT).$(EXT_LIB) $(BCRESULT).cmi $(BCRESULT).cmo \ + $(NCRESULT).cmi $(NCRESULT).cmx $(NCRESULT).o \ + $(RES_CLIB) $(IMPL_CMO:.cmo=.annot) \ + $(LIB_PACK_NAME).cmi $(LIB_PACK_NAME).cmo $(LIB_PACK_NAME).cmx $(LIB_PACK_NAME).o + +ifndef STATIC + NONEXECS += $(DLLSONAME) +endif + +ifndef LIBINSTALL_FILES + LIBINSTALL_FILES := $(RESULT).mli $(RESULT).cmi $(RESULT).cma \ + $(RESULT).cmxa $(RESULT).$(EXT_LIB) $(RES_CLIB) + ifndef STATIC + ifneq ($(strip $(OBJ_LINK)),) + LIBINSTALL_FILES += $(DLLSONAME) + endif + endif +endif + +export LIBINSTALL_FILES + +ifdef WIN32 + # some extra stuff is created while linking DLLs + NONEXECS += $(BCRESULT).$(EXT_LIB) $(BCRESULT).exp $(NCRESULT).exp $(CLIB_BASE).exp $(CLIB_BASE).lib +endif + +TARGETS := $(EXECS) $(NONEXECS) + +# If there are IDL-files +ifneq ($(strip $(FILTERED_IDL)),) + MAYBE_IDL := -cclib -lcamlidl +endif + +ifdef USE_CAMLP4 + CAMLP4PATH := \ + $(shell $(CAMLP4) -where 2>/dev/null || echo /usr/local/lib/camlp4) + INCFLAGS := -I $(CAMLP4PATH) + CINCFLAGS := -I$(CAMLP4PATH) +endif + +DINCFLAGS := $(INCFLAGS) $(SOURCE_DIRS:%=-I %) $(OCAML_DEFAULT_DIRS:%=-I %) +INCFLAGS := $(DINCFLAGS) $(INCDIRS:%=-I %) +CINCFLAGS += $(SOURCE_DIRS:%=-I%) $(INCDIRS:%=-I%) $(OCAML_DEFAULT_DIRS:%=-I%) + +ifndef MSVC + CLIBFLAGS += $(SOURCE_DIRS:%=-L%) $(LIBDIRS:%=-L%) \ + $(EXTLIBDIRS:%=-L%) $(OCAML_DEFAULT_DIRS:%=-L%) + + ifeq ($(ELF_RPATH), yes) + CLIBFLAGS += $(EXTLIBDIRS:%=-Wl,$(RPATH_FLAG)%) + endif +endif + +ifndef PROFILING + INTF_OCAMLC := $(OCAMLC) +else + ifndef THREADS + INTF_OCAMLC := $(OCAMLCP) -p $(OCAMLCPFLAGS) + else + # OCaml does not support profiling byte code + # with threads (yet), therefore we force an error. + ifndef REAL_OCAMLC + $(error Profiling of multithreaded byte code not yet supported by OCaml) + endif + INTF_OCAMLC := $(OCAMLC) + endif +endif + +ifndef MSVC + COMMON_LDFLAGS := $(LDFLAGS:%=-ccopt %) $(SOURCE_DIRS:%=-ccopt -L%) \ + $(LIBDIRS:%=-ccopt -L%) $(EXTLIBDIRS:%=-ccopt -L%) \ + $(EXTLIBDIRS:%=-ccopt -Wl $(OCAML_DEFAULT_DIRS:%=-ccopt -L%)) + + ifeq ($(ELF_RPATH),yes) + COMMON_LDFLAGS += $(EXTLIBDIRS:%=-ccopt -Wl,$(RPATH_FLAG)%) + endif +else + COMMON_LDFLAGS := -ccopt "/link -NODEFAULTLIB:LIBC $(LDFLAGS:%=%) $(SOURCE_DIRS:%=-LIBPATH:%) \ + $(LIBDIRS:%=-LIBPATH:%) $(EXTLIBDIRS:%=-LIBPATH:%) \ + $(OCAML_DEFAULT_DIRS:%=-LIBPATH:%) " +endif + +CLIBS_OPTS := $(CLIBS:%=-cclib -l%) $(CFRAMEWORKS:%=-cclib '-framework %') +ifdef MSVC + ifndef STATIC + # MSVC libraries do not have 'lib' prefix + CLIBS_OPTS := $(CLIBS:%=-cclib %.lib) + endif +endif + +ifneq ($(strip $(OBJ_LINK)),) + ifdef CREATE_LIB + OBJS_LIBS := -cclib -l$(CLIB_BASE) $(CLIBS_OPTS) $(MAYBE_IDL) + else + OBJS_LIBS := $(OBJ_LINK) $(CLIBS_OPTS) $(MAYBE_IDL) + endif +else + OBJS_LIBS := $(CLIBS_OPTS) $(MAYBE_IDL) +endif + +# If we have to make byte-code +ifndef REAL_OCAMLC + BYTE_OCAML := y + + # EXTRADEPS is added dependencies we have to insert for all + # executable files we generate. Ideally it should be all of the + # libraries we use, but it's hard to find the ones that get searched on + # the path since I don't know the paths built into the compiler, so + # just include the ones with slashes in their names. + EXTRADEPS := $(addsuffix .cma,$(foreach i,$(LIBS),$(if $(findstring /,$(i)),$(i)))) + SPECIAL_OCAMLFLAGS := $(OCAMLBCFLAGS) + + REAL_OCAMLC := $(INTF_OCAMLC) + + REAL_IMPL := $(IMPL_CMO) + REAL_IMPL_INTF := $(IMPLO_INTF) + IMPL_SUF := .cmo + + DEPFLAGS := + MAKE_DEPS := $(MLDEPS) $(BCDEPIS) + + ifdef CREATE_LIB + override CFLAGS := $(PIC_CFLAGS) $(CFLAGS) + override CPPFLAGS := $(PIC_CPPFLAGS) $(CPPFLAGS) + ifndef STATIC + ifneq ($(strip $(OBJ_LINK)),) + MAKEDLL := $(DLLSONAME) + ALL_LDFLAGS := -dllib $(DLLSONAME) + endif + endif + endif + + ifndef NO_CUSTOM + ifneq "$(strip $(OBJ_LINK) $(THREADS) $(MAYBE_IDL) $(CLIBS) $(CFRAMEWORKS))" "" + ALL_LDFLAGS += -custom + endif + endif + + ALL_LDFLAGS += $(INCFLAGS) $(OCAMLLDFLAGS) $(OCAMLBLDFLAGS) \ + $(COMMON_LDFLAGS) $(LIBS:%=%.cma) + CAMLIDLDLLFLAGS := + + ifdef THREADS + ifdef VMTHREADS + THREAD_FLAG := -vmthread + else + THREAD_FLAG := -thread + endif + ALL_LDFLAGS := $(THREAD_FLAG) $(ALL_LDFLAGS) + ifndef CREATE_LIB + ifndef REAL_OCAMLFIND + ALL_LDFLAGS := unix.cma threads.cma $(ALL_LDFLAGS) + endif + endif + endif + +# we have to make native-code +else + EXTRADEPS := $(addsuffix .cmxa,$(foreach i,$(LIBS),$(if $(findstring /,$(i)),$(i)))) + ifndef PROFILING + SPECIAL_OCAMLFLAGS := $(OCAMLNCFLAGS) + PLDFLAGS := + else + SPECIAL_OCAMLFLAGS := -p $(OCAMLNCFLAGS) + PLDFLAGS := -p + endif + + REAL_IMPL := $(IMPL_CMX) + REAL_IMPL_INTF := $(IMPLX_INTF) + IMPL_SUF := .cmx + + override CPPFLAGS := -DNATIVE_CODE $(CPPFLAGS) + + DEPFLAGS := -native + MAKE_DEPS := $(MLDEPS) $(NCDEPIS) + + ALL_LDFLAGS := $(PLDFLAGS) $(INCFLAGS) $(OCAMLLDFLAGS) \ + $(OCAMLNLDFLAGS) $(COMMON_LDFLAGS) + CAMLIDLDLLFLAGS := -opt + + ifndef CREATE_LIB + ALL_LDFLAGS += $(LIBS:%=%.cmxa) + else + override CFLAGS := $(PIC_CFLAGS) $(CFLAGS) + override CPPFLAGS := $(PIC_CPPFLAGS) $(CPPFLAGS) + endif + + ifdef THREADS + THREAD_FLAG := -thread + ALL_LDFLAGS := $(THREAD_FLAG) $(ALL_LDFLAGS) + ifndef CREATE_LIB + ifndef REAL_OCAMLFIND + ALL_LDFLAGS := unix.cmxa threads.cmxa $(ALL_LDFLAGS) + endif + endif + endif +endif + +export MAKE_DEPS + +ifdef ANNOTATE + ANNOT_FLAG := -dtypes +else +endif + +ALL_OCAMLCFLAGS := $(THREAD_FLAG) $(ANNOT_FLAG) $(OCAMLFLAGS) \ + $(INCFLAGS) $(SPECIAL_OCAMLFLAGS) + +ifdef make_deps + -include $(MAKE_DEPS) + PRE_TARGETS := +endif + +########################################################################### +# USER RULES + +# Call "OCamlMakefile QUIET=" to get rid of all of the @'s. +QUIET=@ + +# generates byte-code (default) +byte-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(BCRESULT) \ + REAL_RESULT="$(BCRESULT)" make_deps=yes +bc: byte-code + +byte-code-nolink: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) nolink \ + REAL_RESULT="$(BCRESULT)" make_deps=yes +bcnl: byte-code-nolink + +top: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(TOPRESULT) \ + REAL_RESULT="$(BCRESULT)" make_deps=yes + +# generates native-code + +native-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(NCRESULT) \ + REAL_RESULT="$(NCRESULT)" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + make_deps=yes +nc: native-code + +native-code-nolink: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) nolink \ + REAL_RESULT="$(NCRESULT)" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + make_deps=yes +ncnl: native-code-nolink + +# generates byte-code libraries +byte-code-library: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(BCRESULT).cma \ + REAL_RESULT="$(BCRESULT)" \ + CREATE_LIB=yes \ + make_deps=yes +bcl: byte-code-library + +# generates native-code libraries +native-code-library: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(NCRESULT).cmxa \ + REAL_RESULT="$(NCRESULT)" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + CREATE_LIB=yes \ + make_deps=yes +ncl: native-code-library + +ifdef WIN32 +# generates byte-code dll +byte-code-dll: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(BCRESULT).dll \ + REAL_RESULT="$(BCRESULT)" \ + make_deps=yes +bcd: byte-code-dll + +# generates native-code dll +native-code-dll: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(NCRESULT).dll \ + REAL_RESULT="$(NCRESULT)" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + make_deps=yes +ncd: native-code-dll +endif + +# generates byte-code with debugging information +debug-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(BCRESULT) \ + REAL_RESULT="$(BCRESULT)" make_deps=yes \ + OCAMLFLAGS="-g $(OCAMLFLAGS)" \ + OCAMLLDFLAGS="-g $(OCAMLLDFLAGS)" +dc: debug-code + +debug-code-nolink: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) nolink \ + REAL_RESULT="$(BCRESULT)" make_deps=yes \ + OCAMLFLAGS="-g $(OCAMLFLAGS)" \ + OCAMLLDFLAGS="-g $(OCAMLLDFLAGS)" +dcnl: debug-code-nolink + +# generates byte-code with debugging information (native code) +debug-native-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(NCRESULT) \ + REAL_RESULT="$(NCRESULT)" make_deps=yes \ + REAL_OCAMLC="$(OCAMLOPT)" \ + OCAMLFLAGS="-g $(OCAMLFLAGS)" \ + OCAMLLDFLAGS="-g $(OCAMLLDFLAGS)" +dnc: debug-native-code + +debug-native-code-nolink: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) nolink \ + REAL_RESULT="$(NCRESULT)" make_deps=yes \ + REAL_OCAMLC="$(OCAMLOPT)" \ + OCAMLFLAGS="-g $(OCAMLFLAGS)" \ + OCAMLLDFLAGS="-g $(OCAMLLDFLAGS)" +dncnl: debug-native-code-nolink + +# generates byte-code libraries with debugging information +debug-code-library: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(BCRESULT).cma \ + REAL_RESULT="$(BCRESULT)" make_deps=yes \ + CREATE_LIB=yes \ + OCAMLFLAGS="-g $(OCAMLFLAGS)" \ + OCAMLLDFLAGS="-g $(OCAMLLDFLAGS)" +dcl: debug-code-library + +# generates byte-code libraries with debugging information (native code) +debug-native-code-library: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(NCRESULT).cma \ + REAL_RESULT="$(NCRESULT)" make_deps=yes \ + REAL_OCAMLC="$(OCAMLOPT)" \ + CREATE_LIB=yes \ + OCAMLFLAGS="-g $(OCAMLFLAGS)" \ + OCAMLLDFLAGS="-g $(OCAMLLDFLAGS)" +dncl: debug-native-code-library + +# generates byte-code for profiling +profiling-byte-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(BCRESULT) \ + REAL_RESULT="$(BCRESULT)" PROFILING="y" \ + make_deps=yes +pbc: profiling-byte-code + +# generates native-code + +profiling-native-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(NCRESULT) \ + REAL_RESULT="$(NCRESULT)" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + PROFILING="y" \ + make_deps=yes +pnc: profiling-native-code + +# generates byte-code libraries +profiling-byte-code-library: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(BCRESULT).cma \ + REAL_RESULT="$(BCRESULT)" PROFILING="y" \ + CREATE_LIB=yes \ + make_deps=yes +pbcl: profiling-byte-code-library + +# generates native-code libraries +profiling-native-code-library: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(RES_CLIB) $(NCRESULT).cmxa \ + REAL_RESULT="$(NCRESULT)" PROFILING="y" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + CREATE_LIB=yes \ + make_deps=yes +pncl: profiling-native-code-library + +# packs byte-code objects +pack-byte-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) $(BCRESULT).cmo \ + REAL_RESULT="$(BCRESULT)" \ + PACK_LIB=yes make_deps=yes +pabc: pack-byte-code + +# packs native-code objects +pack-native-code: $(PRE_TARGETS) + $(QUIET)$(MAKE) -r -f $(OCAMLMAKEFILE) \ + $(NCRESULT).cmx $(NCRESULT).o \ + REAL_RESULT="$(NCRESULT)" \ + REAL_OCAMLC="$(OCAMLOPT)" \ + PACK_LIB=yes make_deps=yes +panc: pack-native-code + +# generates HTML-documentation +htdoc: $(DOC_DIR)/$(RESULT)/html/index.html + +# generates Latex-documentation +ladoc: $(DOC_DIR)/$(RESULT)/latex/doc.tex + +# generates PostScript-documentation +psdoc: $(DOC_DIR)/$(RESULT)/latex/doc.ps + +# generates PDF-documentation +pdfdoc: $(DOC_DIR)/$(RESULT)/latex/doc.pdf + +# generates all supported forms of documentation +doc: htdoc ladoc psdoc pdfdoc + +########################################################################### +# LOW LEVEL RULES + +$(REAL_RESULT): $(REAL_IMPL_INTF) $(OBJ_LINK) $(EXTRADEPS) $(RESULTDEPS) + $(REAL_OCAMLFIND) $(REAL_OCAMLC) \ + $(OCAML_FIND_PACKAGES) $(OCAML_FIND_LINKPKG) \ + $(ALL_LDFLAGS) $(OBJS_LIBS) -o $@$(EXE) \ + $(REAL_IMPL) + +nolink: $(REAL_IMPL_INTF) $(OBJ_LINK) + +ifdef WIN32 +$(REAL_RESULT).dll: $(REAL_IMPL_INTF) $(OBJ_LINK) + $(CAMLIDLDLL) $(CAMLIDLDLLFLAGS) $(OBJ_LINK) $(CLIBS) \ + -o $@ $(REAL_IMPL) +endif + +%$(TOPSUFFIX): $(REAL_IMPL_INTF) $(OBJ_LINK) $(EXTRADEPS) + $(REAL_OCAMLFIND) $(OCAMLMKTOP) \ + $(OCAML_FIND_PACKAGES) $(OCAML_FIND_LINKPKG) \ + $(ALL_LDFLAGS) $(OBJS_LIBS) -o $@$(EXE) \ + $(REAL_IMPL) + +.SUFFIXES: .mli .ml .cmi .cmo .cmx .cma .cmxa .$(EXT_OBJ) \ + .mly .di .d .$(EXT_LIB) .idl %.oxridl .c .m .$(EXT_CXX) .h .so \ + .rep .zog .glade + +ifndef STATIC +ifdef MINGW +$(DLLSONAME): $(OBJ_LINK) + $(CC) $(CFLAGS) $(CFLAGS_WIN32) $(OBJ_LINK) -shared -o $@ \ + -Wl,--whole-archive $(wildcard $(foreach dir,$(LIBDIRS),$(CLIBS:%=$(dir)/lib%.a))) \ + $(OCAMLLIBPATH)/ocamlrun.a \ + -Wl,--export-all-symbols \ + -Wl,--no-whole-archive +else +ifdef MSVC +$(DLLSONAME): $(OBJ_LINK) + link /NOLOGO /DLL /OUT:$@ $(OBJ_LINK) \ + $(wildcard $(foreach dir,$(LIBDIRS),$(CLIBS:%=$(dir)/%.lib))) \ + $(OCAMLLIBPATH)/ocamlrun.lib + +else +$(DLLSONAME): $(OBJ_LINK) + $(OCAMLMKLIB) $(INCFLAGS) $(CLIBFLAGS) \ + -o $(CLIB_BASE) $(OBJ_LINK) $(CLIBS:%=-l%) $(CFRAMEWORKS:%=-framework %) \ + $(OCAMLMKLIB_FLAGS) +endif +endif +endif + +ifndef LIB_PACK_NAME +$(RESULT).cma: $(REAL_IMPL_INTF) $(MAKEDLL) $(EXTRADEPS) $(RESULTDEPS) + $(REAL_OCAMLFIND) $(REAL_OCAMLC) -a $(ALL_LDFLAGS) $(OBJS_LIBS) -o $@ $(REAL_IMPL) + +$(RESULT).cmxa $(RESULT).$(EXT_LIB): $(REAL_IMPL_INTF) $(EXTRADEPS) $(RESULTDEPS) + $(REAL_OCAMLFIND) $(OCAMLOPT) -a $(ALL_LDFLAGS) $(OBJS_LIBS) -o $@ $(REAL_IMPL) +else +ifdef BYTE_OCAML +$(LIB_PACK_NAME).cmi $(LIB_PACK_NAME).cmo: $(REAL_IMPL_INTF) + $(REAL_OCAMLFIND) $(REAL_OCAMLC) -pack -o $(LIB_PACK_NAME).cmo $(OCAMLLDFLAGS) $(REAL_IMPL) +else +$(LIB_PACK_NAME).cmi $(LIB_PACK_NAME).cmx: $(REAL_IMPL_INTF) + $(REAL_OCAMLFIND) $(OCAMLOPT) -pack -o $(LIB_PACK_NAME).cmx $(OCAMLLDFLAGS) $(REAL_IMPL) +endif + +$(RESULT).cma: $(LIB_PACK_NAME).cmi $(LIB_PACK_NAME).cmo $(MAKEDLL) $(EXTRADEPS) $(RESULTDEPS) + $(REAL_OCAMLFIND) $(REAL_OCAMLC) -a $(ALL_LDFLAGS) $(OBJS_LIBS) -o $@ $(LIB_PACK_NAME).cmo + +$(RESULT).cmxa $(RESULT).$(EXT_LIB): $(LIB_PACK_NAME).cmi $(LIB_PACK_NAME).cmx $(EXTRADEPS) $(RESULTDEPS) + $(REAL_OCAMLFIND) $(OCAMLOPT) -a $(filter-out -custom, $(ALL_LDFLAGS)) $(OBJS_LIBS) -o $@ $(LIB_PACK_NAME).cmx +endif + +$(RES_CLIB): $(OBJ_LINK) +ifndef MSVC + ifneq ($(strip $(OBJ_LINK)),) + $(AR) rcs $@ $(OBJ_LINK) + endif +else + ifneq ($(strip $(OBJ_LINK)),) + lib -nologo -debugtype:cv -out:$(RES_CLIB) $(OBJ_LINK) + endif +endif + +.mli.cmi: $(EXTRADEPS) + $(QUIET)pp=`sed -n -e '/^#/d' -e 's/(\*pp \([^*]*\) \*)/\1/p;q' $<`; \ + if [ -z "$$pp" ]; then \ + $(ECHO) $(REAL_OCAMLFIND) $(INTF_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c $(THREAD_FLAG) $(ANNOT_FLAG) \ + $(OCAMLFLAGS) $(INCFLAGS) $<; \ + $(REAL_OCAMLFIND) $(INTF_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c $(THREAD_FLAG) $(ANNOT_FLAG) \ + $(OCAMLFLAGS) $(INCFLAGS) $<; \ + else \ + $(ECHO) $(REAL_OCAMLFIND) $(INTF_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c -pp \"$$pp $(PPFLAGS)\" $(THREAD_FLAG) $(ANNOT_FLAG) \ + $(OCAMLFLAGS) $(INCFLAGS) $<; \ + $(REAL_OCAMLFIND) $(INTF_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c -pp "$$pp $(PPFLAGS)" $(THREAD_FLAG) $(ANNOT_FLAG) \ + $(OCAMLFLAGS) $(INCFLAGS) $<; \ + fi + +.ml.cmi .ml.$(EXT_OBJ) .ml.cmx .ml.cmo: $(EXTRADEPS) + $(QUIET)pp=`sed -n -e '/^#/d' -e 's/(\*pp \([^*]*\) \*)/\1/p;q' $<`; \ + if [ -z "$$pp" ]; then \ + $(ECHO) $(REAL_OCAMLFIND) $(REAL_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c $(ALL_OCAMLCFLAGS) $<; \ + $(REAL_OCAMLFIND) $(REAL_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c $(ALL_OCAMLCFLAGS) $<; \ + else \ + $(ECHO) $(REAL_OCAMLFIND) $(REAL_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c -pp \"$$pp $(PPFLAGS)\" $(ALL_OCAMLCFLAGS) $<; \ + $(REAL_OCAMLFIND) $(REAL_OCAMLC) $(OCAML_FIND_PACKAGES) \ + -c -pp "$$pp $(PPFLAGS)" $(ALL_OCAMLCFLAGS) $<; \ + fi + +ifdef PACK_LIB +$(REAL_RESULT).cmo $(REAL_RESULT).cmx $(REAL_RESULT).o: $(REAL_IMPL_INTF) $(OBJ_LINK) $(EXTRADEPS) + $(REAL_OCAMLFIND) $(REAL_OCAMLC) -pack $(ALL_LDFLAGS) \ + $(OBJS_LIBS) -o $@ $(REAL_IMPL) +endif + +.PRECIOUS: %.ml +%.ml: %.mll + $(OCAMLLEX) $(LFLAGS) $< + +.PRECIOUS: %.ml %.mli +%.ml %.mli: %.mly + $(OCAMLYACC) $(YFLAGS) $< + $(QUIET)pp=`sed -n -e 's/.*(\*pp \([^*]*\) \*).*/\1/p;q' $<`; \ + if [ ! -z "$$pp" ]; then \ + mv $*.ml $*.ml.temporary; \ + echo "(*pp $$pp $(PPFLAGS)*)" > $*.ml; \ + cat $*.ml.temporary >> $*.ml; \ + rm $*.ml.temporary; \ + mv $*.mli $*.mli.temporary; \ + echo "(*pp $$pp $(PPFLAGS)*)" > $*.mli; \ + cat $*.mli.temporary >> $*.mli; \ + rm $*.mli.temporary; \ + fi + + +.PRECIOUS: %.ml +%.ml: %.rep + $(CAMELEON_REPORT) $(CAMELEON_REPORT_FLAGS) -gen $< + +.PRECIOUS: %.ml +%.ml: %.zog + $(CAMELEON_ZOGGY) $(CAMELEON_ZOGGY_FLAGS) -impl $< > $@ + +.PRECIOUS: %.ml +%.ml: %.glade + $(OCAML_GLADECC) $(OCAML_GLADECC_FLAGS) $< > $@ + +.PRECIOUS: %.ml %.mli +%.ml %.mli: %.oxridl + $(OXRIDL) $< + +.PRECIOUS: %.ml %.mli %_stubs.c %.h +%.ml %.mli %_stubs.c %.h: %.idl + $(CAMLIDL) $(MAYBE_IDL_HEADER) $(IDLFLAGS) \ + $(CAMLIDLFLAGS) $< + $(QUIET)if [ $(NOIDLHEADER) ]; then touch $*.h; fi + +.c.$(EXT_OBJ): + $(OCAMLC) -c -cc "$(CC)" -ccopt "$(CFLAGS) \ + $(CPPFLAGS) $(CPPFLAGS_WIN32) \ + $(CFLAGS_WIN32) $(CINCFLAGS) $(CFLAG_O)$@ " $< + +.m.$(EXT_OBJ): + $(CC) -c $(CFLAGS) $(CINCFLAGS) $(CPPFLAGS) \ + -I'$(OCAMLLIBPATH)' \ + $< $(CFLAG_O)$@ + +.$(EXT_CXX).$(EXT_OBJ): + $(CXX) -c $(CXXFLAGS) $(CINCFLAGS) $(CPPFLAGS) \ + -I'$(OCAMLLIBPATH)' \ + $< $(CFLAG_O)$@ + +$(MLDEPDIR)/%.d: %.ml + $(QUIET)if [ ! -d $(@D) ]; then mkdir -p $(@D); fi + $(QUIET)pp=`sed -n -e '/^#/d' -e 's/(\*pp \([^*]*\) \*)/\1/p;q' $<`; \ + if [ -z "$$pp" ]; then \ + $(ECHO) $(REAL_OCAMLFIND) $(OCAMLDEP) $(OCAML_DEP_PACKAGES) \ + $(DINCFLAGS) $< \> $@; \ + $(REAL_OCAMLFIND) $(OCAMLDEP) $(OCAML_DEP_PACKAGES) \ + $(DINCFLAGS) $< > $@; \ + else \ + $(ECHO) $(REAL_OCAMLFIND) $(OCAMLDEP) $(OCAML_DEP_PACKAGES) \ + -pp \"$$pp $(PPFLAGS)\" $(DINCFLAGS) $< \> $@; \ + $(REAL_OCAMLFIND) $(OCAMLDEP) $(OCAML_DEP_PACKAGES) \ + -pp "$$pp $(PPFLAGS)" $(DINCFLAGS) $< > $@; \ + fi + +$(BCDIDIR)/%.di $(NCDIDIR)/%.di: %.mli + $(QUIET)if [ ! -d $(@D) ]; then mkdir -p $(@D); fi + $(QUIET)pp=`sed -n -e '/^#/d' -e 's/(\*pp \([^*]*\) \*)/\1/p;q' $<`; \ + if [ -z "$$pp" ]; then \ + $(ECHO) $(REAL_OCAMLFIND) $(OCAMLDEP) $(DEPFLAGS) $(DINCFLAGS) $< \> $@; \ + $(REAL_OCAMLFIND) $(OCAMLDEP) $(DEPFLAGS) $(DINCFLAGS) $< > $@; \ + else \ + $(ECHO) $(REAL_OCAMLFIND) $(OCAMLDEP) $(DEPFLAGS) \ + -pp \"$$pp $(PPFLAGS)\" $(DINCFLAGS) $< \> $@; \ + $(REAL_OCAMLFIND) $(OCAMLDEP) $(DEPFLAGS) \ + -pp "$$pp $(PPFLAGS)" $(DINCFLAGS) $< > $@; \ + fi + +$(DOC_DIR)/$(RESULT)/html: + mkdir -p $@ + +$(DOC_DIR)/$(RESULT)/html/index.html: $(DOC_DIR)/$(RESULT)/html $(DOC_FILES) + rm -rf $ + +module Numberz = +struct +type t = +| ONE +| TWO +| THREE +| FIVE +| SIX +| EIGHT + +let of_i = ... +let to_i = ... +end + +typedef format +-------------- +Typedef turns into the type declaration: +typedef i64 UserId + +==> + +type userid Int64.t + +exception format +---------------- +The same as structs except that the module also has an exception type +E of t that is raised/caught. + +For example, with an exception Xception, +raise (Xception.E (new Xception.t)) +and +try + ... +with Xception.E e -> ... + +list format +----------- +Lists are turned into OCaml native lists. + +Map/Set formats +--------------- +These are both turned into Hashtbl.t's. Set values are bool. + +Services +-------- +The client is a class "client" parametrized on input and output +protocols. The processor is a class parametrized on a handler. A +handler is a class inheriting the iface abstract class. Unlike other +implementations, client does not implement iface since iface functions +must take option arguments so as to deal with the case where a client +does not send all the arguments. diff --git a/lib/ocaml/README-OCamlMakefile b/lib/ocaml/README-OCamlMakefile new file mode 100644 index 00000000..54787b96 --- /dev/null +++ b/lib/ocaml/README-OCamlMakefile @@ -0,0 +1,640 @@ +--------------------------------------------------------------------------- + + Distribution of "ocaml_make" + Copyright (C) 1999 - 2006 Markus Mottl - free to copy and modify! + USE AT YOUR OWN RISK! + +--------------------------------------------------------------------------- + + PREREQUISITES + + *** YOU WILL NEED GNU-MAKE VERSION >3.80 *** + +--------------------------------------------------------------------------- + + Contents of this distribution + +Changes - guess what? ;-) + +OCamlMakefile - Makefile for easy handling of compilation of not so easy + OCaml-projects. It generates dependencies of OCaml-files + automatically, is able to handle "ocamllex"-, + "ocamlyacc"-, IDL- and C-files, knows how to run + preprocessors and generates native- or byte-code, as + executable or as library - with thread-support if you + want! Profiling and debugging support can be added on + the fly! There is also support for installing libraries. + Ah, yes, and you can also create toplevels from any + sources: this allows you immediate interactive testing. + Automatic generation of documentation is easy due to + integration of support for OCamldoc. + +README - this file + +calc/ - Directory containing a quite fully-featured example + of what "OCamlMakefile" can do for you. This example + makes use of "ocamllex", "ocamlyacc", IDL + C and + threads. + +camlp4/ - This simple example demonstrates how to automatically + preprocess files with the camlp4-preprocessor. + +gtk/ - Demonstration of how to use OCamlMakefile with GTK + and threads. Courtesy of Tim Freeman . + +idl/ - Contains a very small example of how to use + "camlidl" together with "OCamlMakefile". Also intended + to show, how easy it is to interface OCaml and C. + +threads/ - Two examples of how to use threads (originally + posted by Xavier Leroy some time ago). Shows the use of + "OCamlMakefile" in an environment of multiple compilation + targets. + +--------------------------------------------------------------------------- + + Why should you use it? + +For several reasons: + + * It is well-tested (I use it in all of my projects). + + * In contrast to most other approaches it generates dependencies + correctly by ensuring that all automatically generated OCaml-files + exist before dependency calculation. This is the only way to + guarantee that "ocamldep" works correctly. + + * It is extremely convenient (at least I think so ;-). + Even quite complex compilation processes (see example "calc.ml") + need very little information to work correctly - actually just about + the minimum (file names of sources). + +--------------------------------------------------------------------------- + + When you shouldn't use it... + +In projects where every compilation unit needs different flags - but +in such complicated cases you will be on your own anyway. Luckily, +this doesn't happen too frequently... + +--------------------------------------------------------------------------- + + How to use "OCamlMakefile" in your own project + (Take a look at the examples for a quick introduction!) + +Create your project-specific "Makefile" in the appropriate directory. + +Now there are two ways of making use of "OCamlMakefile": + + 1) Have a look at the default settings in "OCamlMakefile" and set + them to the values that are vaild on your system - whether the + path to the standard libraries is ok, what executables shall be + used, etc... + + 2) Copy it into the directory of the project to be compiled. + Add "-include OCamlMakefile" as a last line of your "Makefile". + + 3) Put it somewhere else on the system. In this case you will have to + set a variable "OCAMLMAKEFILE" in your project-specific "Makefile". + This is the way in which the examples are written: so you need + only one version of "OCamlMakefile" to manage all your projects! + See the examples for details. + +You should usually specify two further variables for your project: + + * SOURCES (default: foo.ml) + * RESULT (default: foo) + +Put all the sources necessary for a target into variable "SOURCES". +Then set "RESULT" to the name of the target. If you want to generate +libraries, you should *not* specify the suffix (".cma", ".cmxa", ".a") +- it will be added automatically if you specify that you want to build +a library. + + ** Don't forget to add the ".mli"-files, too! ** + ** Don't forget that order of the source files matters! ** + +The order is important, because it matters during linking anyway +due to potential side effects caused at program startup. This is +why OCamlMakefile does not attempt to partially order dependencies by +itself, which might confuse users even more. It just compiles and links +OCaml-sources in the order specified by the user, even if it could +determine automatically that the order cannot be correct. + +The minimum of your "Makefile" looks like this (assuming that +"OCamlMakefile" is in the search path of "make"): + + -include OCamlMakefile + +This will assume that you want to compile a file "foo.ml" to a binary +"foo". + +Otherwise, your Makefile will probably contain something like this: + + SOURCES = foo.ml + RESULT = foo + -include OCamlMakefile + +Be careful with the names you put into these variables: if they are wrong, +a "make clean" might erase the wrong files - but I know you will not do +that ;-) + +A simple "make" will generate a byte-code executable. If you want to +change this, you may add an "all"-rule that generates something else. + +E.g.: + + SOURCES = foo.ml + RESULT = foo + all: native-code-library + -include OCamlMakefile + +This will build a native-code library "foo.cmxa" (+ "foo.a") from file +"foo.ml". + +You may even build several targets at once. To produce byte- and native-code +executables with one "make", add the following rule: + + all: byte-code native-code + +You will probably want to use a different suffix for each of these targets +so that the result will not be overwritten (see optional variables below +for details). + +You may also tell "make" at the command-line what kind of target to +produce (e.g. "make nc"). Here all the possibilities with shortcuts +between parenthesis: + + * byte-code (bc) + * byte-code-nolink (bcnl) - no linking stage + * byte-code-library (bcl) + * native-code (nc) + * native-code-nolink (ncnl) - no linking stage + * native-code-library (ncl) + * debug-code (dc) + * debug-code-nolink (dcnl) - no linking stage + * debug-code-library (dcl) + * profiling-byte-code (pbc) + * profiling-byte-code-library (pbcl) + * profiling-native-code (pnc) + * profiling-native-code-library (pncl) + * byte-code-dll (bcd) + * native-code-dll (ncd) + * pack-byte-code (pabc) + * pack-native-code (panc) + * toplevel interpreter (top) + * subprojs + +Here a short note concerning building and linking byte code libraries +with C-files: + + OCaml links C-object files only when they are used in an executable. + After compilation they should be placed in some directory that is in + your include path if you link your library against an executable. + + It is sometimes more convenient to link all C-object files into a + single C-library. Then you have to override the automatic link flags + of your library using "-noautolink" and add another linkflag that + links in your C-library explicitly. + +What concerns maintainance: + + "make clean" removes all (all!) automatically generated files - so + again: make sure your variables are ok! + + "make cleanup" is similar to "make clean" but leaves executables. + +Another way to destroy some important files is by having "OCamlMakefile" +automatically generate files with the same name. Read the documentation +about the tools in the OCaml-distribution to see what kind of files are +generated. "OCamlMakefile" additionally generates ('%' is basename of +source file): + + %_idl.c - "camlidl" generates a file "%.c" from "%.idl", but this is + not such a good idea, because when generating native-code, + both the file "%.c" and "%.ml" would generate files "%.o" + which would overwrite each other. Thus, "OCamlMakefile" + renames "%.c" to "%_idl.c" to work around this problem. + +The dependencies are stored in three different subdirectories (dot dirs): + + ._d - contains dependencies for .ml-files + ._bcdi - contains byte code dependencies for .mli-files + ._ncdi - contains native code dependencies for .mli-files + +The endings of the dependency files are: "%.d" for those generated from +"%.ml"-files, "%.di" for ones derived from "%.mli"-files. + +--------------------------------------------------------------------------- + + Debugging + + This is easy: if you discover a bug, just do a "make clean; make dc" + to recompile your project with debugging information. Then you can + immediately apply "ocamldebug" to the executable. + +--------------------------------------------------------------------------- + + Profiling + + For generating code that can be profiled with "ocamlprof" (byte code) + or "gprof" (native code), compile your project with one of the profiling + targets (see targets above). E.g.: + + * "make pbc" will build byte code that can be profiled with + "ocamlprof". + + * "make pnc" will build native code that can be profiled with + "gprof". + + Please note that it is not currently possible to profile byte code with + threads. OCamlMakefile will force an error if you try to do this. + + A short hint for DEC Alpha-users (under Digital Unix): you may also + compile your sources to native code without any further profiling + options/targets. Then call "pixie my_exec", "my_exec" being your + executable. This will produce (among other files) an executable + "my_exec.pixie". Call it and it will produce profiling information which + can be analysed using "prof -pixie my_exec". The resulting information + is extremely detailed and allows analysis up to the clock cycle level... + +--------------------------------------------------------------------------- + + Using Preprocessors + + Because one could employ any kind of program that reads from standard + input and prints to standard output as preprocessor, there cannot be any + default way to handle all of them correctly without further knowledge. + + Therefore you have to cooperate a bit with OCamlMakefile to let + preprocessing happen automatically. Basically, this only requires + that you put a comment into the first line of files that should be + preprocessed, e.g.: + + (*pp cat *) + ... rest of program ... + + OCamlMakefile looks at the first line of your files, and if it finds + a comment that starts with "(*pp", then it will assume that the + rest of the comment tells it how to correctly call the appropriate + preprocessor. In this case the program "cat" will be called, which will, + of course, just output the source text again without changing it. + + If you are, for example, an advocate of the new "revised syntax", + which is supported by the camlp4 preprocessor, you could simply write: + + (*pp camlp4r *) + ... rest of program in revised syntax ... + + Simple, isn't it? + + If you want to write your own syntax extensions, just take a look at the + example in the directory "camlp4": it implements the "repeat ... until" + extension as described in the camlp4-tutorial. + +--------------------------------------------------------------------------- + + Library (Un-)Installation Support + + OCamlMakefile contains two targets using "ocamlfind" for this purpose: + + * libinstall + * libuninstall + + These two targets require the existence of the variable + "LIBINSTALL_FILES", which should be set to all the files that you + want to install in the library directory (usually %.mli, %.cmi, %.cma, + %.cmxa, %.a and possibly further C-libraries). The target "libinstall" + has the dependency "all" to force compilation of the library so make + sure you define target "all" in your Makefile appropriately. + + The targets inform the user about the configured install path and ask + for confirmation to (un)install there. If you want to use them, it + is often a good idea to just alias them in your Makefile to "install" + and "uninstall" respectively. + + Two other targets allow installation of files into a particular + directory (without using ocamlfind): + + * rawinstall + * rawuninstall + +--------------------------------------------------------------------------- + + Building toplevels + + There is just one target for this: + + * top + + The generated file can be used immediately for interactive sessions - + even with scanners, parsers, C-files, etc.! + +--------------------------------------------------------------------------- + + Generating documentation + + The following targets are supported: + + * htdoc - generates HTML-documentation + * ladoc - generates Latex-documentation + * psdoc - generates PostScript-documentation + * pdfdoc - generates PDF-documentation + * doc - generates all supported forms of documentation + * clean-doc - generates all supported forms of documentation + + All of them generate a sub-directory "doc". More precisely, for HTML it + is "doc/$(RESULT)/html" and for Latex, PostScript and PDF the directory + "doc/$(RESULT)/latex". See the OCamldoc-manual for details and the + optional variables below for settings you can control. + +--------------------------------------------------------------------------- + + Handling subprojects + + You can have several targets in the same directory and manage them + from within an single Makefile. + + Give each subproject a name, e.g. "p1", "p2", etc. Then you export + settings specific to each project by using variables of the form + "PROJ_p1", "PROJ_p2", etc. E.g.: + + define PROJ_p1 + SOURCES="foo.ml main.ml" + RESULT="p1" + OCAMLFLAGS="-unsafe" + endef + export PROJ_p1 + + define PROJ_p2 + ... + endef + export PROJ_p2 + + You may also export common settings used by all projects directly, e.g. + "export THREADS = y". + + Now it is a good idea to define, which projects should be affected by + commands by default. E.g.: + + ifndef SUBPROJS + export SUBPROJS = p1 p2 + endif + + This will automatically generate a given target for all those + subprojects if this variable has not been defined in the shell + environment or in the command line of the make-invocation by the user. + E.g., "make dc" will generate debug code for all subprojects. + + Then you need to define a default action for your subprojects if "make" + has been called without arguments: + + all: bc + + This will build byte code by default for all subprojects. + + Finally, you'll have to define a catch-all target that uses the target + provided by the user for all subprojects. Just add (assuming that + OCAMLMAKEFILE has been defined appropriately): + + %: + @make -f $(OCAMLMAKEFILE) subprojs SUBTARGET=$@ + + See the "threads"-directory in the distribution for a short example! + +--------------------------------------------------------------------------- + + Optional variables that may be passed to "OCamlMakefile" + + * LIB_PACK_NAME - packs all modules of a library into a module whose + name is given in variable "LIB_PACK_NAME". + + * RES_CLIB_SUF - when building a library that contains C-stubs, this + variable controls the suffix appended to the name + of the C-library (default: "_stubs"). + + * THREADS - say "THREADS = yes" if you need thread support compiled in, + otherwise leave it away. + + * VMTHREADS - say "VMTHREADS = yes" if you want to force VM-level + scheduling of threads (byte-code only). + + * ANNOTATE - say "ANNOTATE = yes" to generate type annotation files + (.annot) to support displaying of type information + in editors. + + * USE_CAMLP4 - say "USE_CAMLP4 = yes" in your "Makefile" if you + want to include the camlp4 directory during the + build process, otherwise leave it away. + + * INCDIRS - directories that should be searched for ".cmi"- and + ".cmo"-files. You need not write "-I ..." - just the + plain names. + * LIBDIRS - directories that should be searched for libraries + Also just put the plain paths into this variable + * EXTLIBDIRS - Same as "LIBDIRS", but paths in this variable are + also added to the binary via the "-R"-flag so that + dynamic libraries in non-standard places can be found. + * RESULTDEPS - Targets on which results (executables or libraries) + should additionally depend. + + * PACKS - adds packages under control of "findlib". + + * PREDS - specifies "findlib"-predicates. + + * LIBS - OCaml-libraries that should be linked (just plain names). + E.g. if you want to link the Str-library, just write + "str" (without quotes). + The new OCaml-compiler handles libraries in such + a way that they "remember" whether they have to + be linked against a C-library and it gets linked + in automatically. + If there is a slash in the library name (such as + "./str" or "lib/foo") then make is told that the + generated files depend on the library. This + helps to ensure that changes to your libraries are + taken into account, which is important if you are + regenerating your libraries frequently. + * CLIBS - C-libraries that should be linked (just plain names). + + * PRE_TARGETS - set this to a list of target files that you want + to have buildt before dependency calculation actually + takes place. E.g. use this to automatically compile + modules needed by camlp4, which have to be available + before other modules can be parsed at all. + + ** WARNING **: the files mentioned in this variable + will be removed when "make clean" is executed! + + * LIBINSTALL_FILES - the files of a library that should be installed + using "findlib". Default: + + $(RESULT).mli $(RESULT).cmi $(RESULT).cma + $(RESULT).cmxa $(RESULT).a lib$(RESULT).a + + * OCAML_LIB_INSTALL - target directory for "rawinstall/rawuninstall". + (default: $(OCAMLLIBPATH)/contrib) + + * DOC_FILES - names of files from which documentation is generated. + (default: all .mli-files in your $(SOURCES)). + + * DOC_DIR - name of directory where documentation should be stored. + + * OCAMLFLAGS - flags passed to the compilers + * OCAMLBCFLAGS - flags passed to the byte code compiler only + * OCAMLNCFLAGS - flags passed to the native code compiler only + + * OCAMLLDFLAGS - flags passed to the OCaml-linker + * OCAMLBLDFLAGS - flags passed to the OCaml-linker when linking byte code + * OCAMLNLDFLAGS - flags passed to the OCaml-linker when linking + native code + + * OCAMLMKLIB_FLAGS - flags passed to the OCaml library tool + + * OCAMLCPFLAGS - profiling flags passed to "ocamlcp" (default: "a") + + * PPFLAGS - additional flags passed to the preprocessor (default: none) + + * LFLAGS - flags passed to "ocamllex" + * YFLAGS - flags passed to "ocamlyacc" + * IDLFLAGS - flags passed to "camlidl" + + * OCAMLDOCFLAGS - flags passed to "ocamldoc" + + * OCAMLFIND_INSTFLAGS - flags passed to "ocamlfind" during installation + (default: none) + + * DVIPSFLAGS - flags passed to dvips + (when generating documentation in PostScript). + + * STATIC - set this variable if you want to force creation + of static libraries + + * CC - the C-compiler to be used + * CXX - the C++-compiler to be used + + * CFLAGS - additional flags passed to the C-compiler. + The flag "-DNATIVE_CODE" will be passed automatically + if you choose to build native code. This allows you + to compile your C-files conditionally. But please + note: You should do a "make clean" or remove the + object files manually or touch the %.c-files: + otherwise, they may not be correctly recompiled + between different builds. + + * CXXFLAGS - additional flags passed to the C++-compiler. + + * CPPFLAGS - additional flags passed to the C-preprocessor. + + * CFRAMEWORKS - Objective-C framework to pass to linker on MacOS X. + + * LDFLAGS - additional flags passed to the C-linker + + * RPATH_FLAG - flag passed through to the C-linker to set a path for + dynamic libraries. May need to be set by user on + exotic platforms. (default: "-R"). + + * ELF_RPATH_FLAG - this flag is used to set the rpath on ELF-platforms. + (default: "-R") + + * ELF_RPATH - if this flag is "yes", then the RPATH_FLAG will be + passed by "-Wl" to the linker as normal on + ELF-platforms. + + * OCAMLLIBPATH - path to the OCaml-standard-libraries + (first default: `$(OCAMLC) -where`) + (second default: "/usr/local/lib/ocaml") + + * OCAML_DEFAULT_DIRS - additional path in which the user can supply + default directories to his own collection of + libraries. The idea is to pass this as an environment + variable so that the Makefiles do not have to contain + this path all the time. + + * OCAMLFIND - ocamlfind from findlib (default: "ocamlfind") + * OCAMLC - byte-code compiler (default: "ocamlc") + * OCAMLOPT - native-code compiler (default: "ocamlopt") + * OCAMLMKTOP - top-level compiler (default: "ocamlmktop") + * OCAMLCP - profiling byte-code compiler (default: "ocamlcp") + * OCAMLDEP - dependency generator (default: "ocamldep") + * OCAMLLEX - scanner generator (default: "ocamllex") + * OCAMLYACC - parser generator (default: "ocamlyacc") + * OCAMLMKLIB - tool to create libraries (default: "ocamlmklib") + * CAMLIDL - IDL-code generator (default: "camlidl") + * CAMLIDLDLL - IDL-utility (default: "camlidldll") + * CAMLP4 - camlp4 preprocessor (default: "camlp4") + * OCAMLDOC - OCamldoc-command (default: "ocamldoc") + + * LATEX - Latex-processor (default: "latex") + * DVIPS - dvips-command (default: "dvips") + * PS2PDF - PostScript-to-PDF converter (default: "ps2pdf") + + * CAMELEON_REPORT - report tool of Cameleon (default: "report") + * CAMELEON_REPORT_FLAGS - flags for the report tool of Cameleon + + * CAMELEON_ZOGGY - zoggy tool of Cameleon + (default: "camlp4o pa_zog.cma pr_o.cmo") + * CAMELEON_ZOGGY_FLAGS - flags for the zoggy tool of Cameleon + + * OCAML_GLADECC - Glade compiler for OCaml (default: "lablgladecc2") + * OCAML_GLADECC_FLAGS - flags for the Glade compiler + + * OXRIDL - OXRIDL-generator (default: "oxridl") + + * NOIDLHEADER - set to "yes" to prohibit "OCamlMakefile" from using + the default camlidl-flag "-header". + + * NO_CUSTOM - Prevent linking in custom mode. + + * QUIET - unsetting this variable (e.g. "make QUIET=") + will print all executed commands, including + intermediate ones. This allows more comfortable + debugging when things go wrong during a build. + + * REALLY_QUIET - when set this flag turns off output from some commands. + + * OCAMLMAKEFILE - location of (=path to) this "OCamlMakefile". + Because it calles itself recursively, it has to + know where it is. (default: "OCamlMakefile" = + local directory) + + * BCSUFFIX - Suffix for all byte-code files. E.g.: + + RESULT = foo + BCSUFFIX = _bc + + This will produce byte-code executables/libraries + with basename "foo_bc". + + * NCSUFFIX - Similar to "BCSUFFIX", but for native-code files. + * TOPSUFFIX - Suffix added to toplevel interpreters (default: ".top") + + * SUBPROJS - variable containing the names of subprojects to be + compiled. + + * SUBTARGET - target to be built for all projects in variable + SUBPROJS. + +--------------------------------------------------------------------------- + + Optional variables for Windows users + + * MINGW - variable to detect the MINGW-environment + * MSVC - variable to detect the MSVC-compiler + +--------------------------------------------------------------------------- + +Up-to-date information (newest release of distribution) can always be +found at: + + http://www.ocaml.info/home/ocaml_sources.html + +--------------------------------------------------------------------------- + +Enjoy! + +New York, 2007-04-22 +Markus Mottl + +e-mail: markus.mottl@gmail.com +WWW: http://www.ocaml.info diff --git a/lib/ocaml/TODO b/lib/ocaml/TODO new file mode 100644 index 00000000..4d1dc771 --- /dev/null +++ b/lib/ocaml/TODO @@ -0,0 +1,5 @@ +Write interfaces +Clean up the code generator +Avoid capture properly instead of relying on the user not to use _ + + diff --git a/lib/ocaml/src/Makefile b/lib/ocaml/src/Makefile new file mode 100644 index 00000000..42ec8dbd --- /dev/null +++ b/lib/ocaml/src/Makefile @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +SOURCES = Thrift.ml TBinaryProtocol.ml TSocket.ml TChannelTransport.ml TServer.ml TSimpleServer.ml TServerSocket.ml TThreadedServer.ml +RESULT = thrift +LIBS = unix threads +THREADS = yes +all: native-code-library debug-code-library top +OCAMLMAKEFILE = ../OCamlMakefile +include $(OCAMLMAKEFILE) diff --git a/lib/ocaml/src/TBinaryProtocol.ml b/lib/ocaml/src/TBinaryProtocol.ml new file mode 100644 index 00000000..a06cc9a9 --- /dev/null +++ b/lib/ocaml/src/TBinaryProtocol.ml @@ -0,0 +1,171 @@ +(* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*) + +open Thrift + +module P = Protocol + +let get_byte i b = 255 land (i lsr (8*b)) +let get_byte64 i b = 255 land (Int64.to_int (Int64.shift_right i (8*b))) + + +let tv = P.t_type_to_i +let vt = P.t_type_of_i + + +let comp_int b n = + let s = ref 0l in + let sb = 32 - 8*n in + for i=0 to (n-1) do + s:= Int32.logor !s (Int32.shift_left (Int32.of_int (int_of_char b.[i])) (8*(n-1-i))) + done; + Int32.to_int (Int32.shift_right (Int32.shift_left !s sb) sb) + +let comp_int64 b n = + let s = ref 0L in + for i=0 to (n-1) do + s:=Int64.logor !s (Int64.shift_left (Int64.of_int (int_of_char b.[i])) (8*(n-1-i))) + done; + !s + +let version_mask = 0xffff0000 +let version_1 = 0x80010000 + +class t trans = +object (self) + inherit P.t trans + val ibyte = String.create 8 + method writeBool b = + ibyte.[0] <- char_of_int (if b then 1 else 0); + trans#write ibyte 0 1 + method writeByte i = + ibyte.[0] <- char_of_int (get_byte i 0); + trans#write ibyte 0 1 + method writeI16 i = + let gb = get_byte i in + ibyte.[1] <- char_of_int (gb 0); + ibyte.[0] <- char_of_int (gb 1); + trans#write ibyte 0 2 + method writeI32 i = + let gb = get_byte i in + for i=0 to 3 do + ibyte.[3-i] <- char_of_int (gb i) + done; + trans#write ibyte 0 4 + method writeI64 i= + let gb = get_byte64 i in + for i=0 to 7 do + ibyte.[7-i] <- char_of_int (gb i) + done; + trans#write ibyte 0 8 + method writeDouble d = + self#writeI64 (Int64.bits_of_float d) + method writeString s= + let n = String.length s in + self#writeI32(n); + trans#write s 0 n + method writeBinary a = self#writeString a + method writeMessageBegin (n,t,s) = + self#writeI32 (version_1 lor (P.message_type_to_i t)); + self#writeString n; + self#writeI32 s + method writeMessageEnd = () + method writeStructBegin s = () + method writeStructEnd = () + method writeFieldBegin (n,t,i) = + self#writeByte (tv t); + self#writeI16 i + method writeFieldEnd = () + method writeFieldStop = + self#writeByte (tv (Protocol.T_STOP)) + method writeMapBegin (k,v,s) = + self#writeByte (tv k); + self#writeByte (tv v); + self#writeI32 s + method writeMapEnd = () + method writeListBegin (t,s) = + self#writeByte (tv t); + self#writeI32 s + method writeListEnd = () + method writeSetBegin (t,s) = + self#writeByte (tv t); + self#writeI32 s + method writeSetEnd = () + method readByte = + ignore (trans#readAll ibyte 0 1); + (comp_int ibyte 1) + method readI16 = + ignore (trans#readAll ibyte 0 2); + comp_int ibyte 2 + method readI32 = + ignore (trans#readAll ibyte 0 4); + comp_int ibyte 4 + method readI64 = + ignore (trans#readAll ibyte 0 8); + comp_int64 ibyte 8 + method readDouble = + Int64.float_of_bits (self#readI64) + method readBool = + self#readByte = 1 + method readString = + let sz = self#readI32 in + let buf = String.create sz in + ignore (trans#readAll buf 0 sz); + buf + method readBinary = self#readString + method readMessageBegin = + let ver = self#readI32 in + if (ver land version_mask != version_1) then + (print_int ver; + raise (P.E (P.BAD_VERSION, "Missing version identifier"))) + else + let s = self#readString in + let mt = P.message_type_of_i (ver land 0xFF) in + (s,mt, self#readI32) + method readMessageEnd = () + method readStructBegin = + "" + method readStructEnd = () + method readFieldBegin = + let t = (vt (self#readByte)) + in + if t != P.T_STOP then + ("",t,self#readI16) + else ("",t,0); + method readFieldEnd = () + method readMapBegin = + let kt = vt (self#readByte) in + let vt = vt (self#readByte) in + (kt,vt, self#readI32) + method readMapEnd = () + method readListBegin = + let t = vt (self#readByte) in + (t,self#readI32) + method readListEnd = () + method readSetBegin = + let t = vt (self#readByte) in + (t, self#readI32); + method readSetEnd = () +end + +class factory = +object + inherit P.factory + method getProtocol tr = new t tr +end diff --git a/lib/ocaml/src/TChannelTransport.ml b/lib/ocaml/src/TChannelTransport.ml new file mode 100644 index 00000000..0f7d616f --- /dev/null +++ b/lib/ocaml/src/TChannelTransport.ml @@ -0,0 +1,39 @@ +(* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*) + +open Thrift +module T = Transport + +class t (i,o) = +object (self) + val mutable opened = true + inherit Transport.t + method isOpen = opened + method opn = () + method close = close_in i; opened <- false + method read buf off len = + if opened then + try + really_input i buf off len; len + with _ -> raise (T.E (T.UNKNOWN, ("TChannelTransport: Could not read "^(string_of_int len)))) + else + raise (T.E (T.NOT_OPEN, "TChannelTransport: Channel was closed")) + method write buf off len = output o buf off len + method flush = flush o +end diff --git a/lib/ocaml/src/TServer.ml b/lib/ocaml/src/TServer.ml new file mode 100644 index 00000000..fc51efa8 --- /dev/null +++ b/lib/ocaml/src/TServer.ml @@ -0,0 +1,42 @@ +(* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*) + +open Thrift + +class virtual t + (pf : Processor.t) + (st : Transport.server_t) + (tf : Transport.factory) + (ipf : Protocol.factory) + (opf : Protocol.factory)= +object + method virtual serve : unit +end;; + + + +let run_basic_server proc port = + Unix.establish_server (fun inp -> fun out -> + let trans = new TChannelTransport.t (inp,out) in + let proto = new TBinaryProtocol.t (trans :> Transport.t) in + try + while proc#process proto proto do () done; () + with e -> ()) (Unix.ADDR_INET (Unix.inet_addr_of_string "127.0.0.1",port)) + + diff --git a/lib/ocaml/src/TServerSocket.ml b/lib/ocaml/src/TServerSocket.ml new file mode 100644 index 00000000..405ef82c --- /dev/null +++ b/lib/ocaml/src/TServerSocket.ml @@ -0,0 +1,41 @@ +(* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*) + +open Thrift + +class t port = +object + inherit Transport.server_t + val mutable sock = None + method listen = + let s = Unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in + sock <- Some s; + Unix.bind s (Unix.ADDR_INET (Unix.inet_addr_any, port)); + Unix.listen s 256 + method close = + match sock with + Some s -> Unix.shutdown s Unix.SHUTDOWN_ALL; Unix.close s; + sock <- None + | _ -> () + method acceptImpl = + match sock with + Some s -> let (fd,_) = Unix.accept s in + new TChannelTransport.t (Unix.in_channel_of_descr fd,Unix.out_channel_of_descr fd) + | _ -> raise (Transport.E (Transport.NOT_OPEN,"TServerSocket: Not listening but tried to accept")) +end diff --git a/lib/ocaml/src/TSimpleServer.ml b/lib/ocaml/src/TSimpleServer.ml new file mode 100644 index 00000000..d19d8c55 --- /dev/null +++ b/lib/ocaml/src/TSimpleServer.ml @@ -0,0 +1,38 @@ +(* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*) + +open Thrift +module S = TServer + +class t pf st tf ipf opf = +object + inherit S.t pf st tf ipf opf + method serve = + try + st#listen; + let c = st#accept in + let trans = tf#getTransport c in + let inp = ipf#getProtocol trans in + let op = opf#getProtocol trans in + try + while (pf#process inp op) do () done; + trans#close + with e -> trans#close; raise e + with _ -> () +end diff --git a/lib/ocaml/src/TSocket.ml b/lib/ocaml/src/TSocket.ml new file mode 100644 index 00000000..109e11c5 --- /dev/null +++ b/lib/ocaml/src/TSocket.ml @@ -0,0 +1,59 @@ +(* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*) + +open Thrift + +module T = Transport + +class t host port= +object (self) + inherit T.t + val mutable chans = None + method isOpen = chans != None + method opn = + try + let addr = (let {Unix.h_addr_list=x} = Unix.gethostbyname host in x.(0)) in + chans <- Some(Unix.open_connection (Unix.ADDR_INET (addr,port))) + with + Unix.Unix_error (e,fn,_) -> raise (T.E (T.NOT_OPEN, ("TSocket: Could not connect to "^host^":"^(string_of_int port)^" because: "^fn^":"^(Unix.error_message e)))) + | _ -> raise (T.E (T.NOT_OPEN, ("TSocket: Could not connect to "^host^":"^(string_of_int port)))) + + method close = + match chans with + None -> () + | Some(inc,out) -> (Unix.shutdown_connection inc; + close_in inc; + chans <- None) + method read buf off len = match chans with + None -> raise (T.E (T.NOT_OPEN, "TSocket: Socket not open")) + | Some(i,o) -> + try + really_input i buf off len; len + with + Unix.Unix_error (e,fn,_) -> raise (T.E (T.UNKNOWN, ("TSocket: Could not read "^(string_of_int len)^" from "^host^":"^(string_of_int port)^" because: "^fn^":"^(Unix.error_message e)))) + | _ -> raise (T.E (T.UNKNOWN, ("TSocket: Could not read "^(string_of_int len)^" from "^host^":"^(string_of_int port)))) + method write buf off len = match chans with + None -> raise (T.E (T.NOT_OPEN, "TSocket: Socket not open")) + | Some(i,o) -> output o buf off len + method flush = match chans with + None -> raise (T.E (T.NOT_OPEN, "TSocket: Socket not open")) + | Some(i,o) -> flush o +end + + diff --git a/lib/ocaml/src/TThreadedServer.ml b/lib/ocaml/src/TThreadedServer.ml new file mode 100644 index 00000000..4462dbd7 --- /dev/null +++ b/lib/ocaml/src/TThreadedServer.ml @@ -0,0 +1,45 @@ +(* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*) + +open Thrift + +class t + (pf : Processor.t) + (st : Transport.server_t) + (tf : Transport.factory) + (ipf : Protocol.factory) + (opf : Protocol.factory)= +object + inherit TServer.t pf st tf ipf opf + method serve = + st#listen; + while true do + let tr = tf#getTransport (st#accept) in + ignore (Thread.create + (fun _ -> + let ip = ipf#getProtocol tr in + let op = opf#getProtocol tr in + try + while pf#process ip op do + () + done + with _ -> ()) ()) + done +end + diff --git a/lib/ocaml/src/Thrift.ml b/lib/ocaml/src/Thrift.ml new file mode 100644 index 00000000..8dc9afa3 --- /dev/null +++ b/lib/ocaml/src/Thrift.ml @@ -0,0 +1,368 @@ +(* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*) + +exception Break;; +exception Thrift_error;; +exception Field_empty of string;; + +class t_exn = +object + val mutable message = "" + method get_message = message + method set_message s = message <- s +end;; + +module Transport = +struct + type exn_type = + | UNKNOWN + | NOT_OPEN + | ALREADY_OPEN + | TIMED_OUT + | END_OF_FILE;; + + exception E of exn_type * string + + class virtual t = + object (self) + method virtual isOpen : bool + method virtual opn : unit + method virtual close : unit + method virtual read : string -> int -> int -> int + method readAll buf off len = + let got = ref 0 in + let ret = ref 0 in + while !got < len do + ret := self#read buf (off+(!got)) (len - (!got)); + if !ret <= 0 then + raise (E (UNKNOWN, "Cannot read. Remote side has closed.")); + got := !got + !ret + done; + !got + method virtual write : string -> int -> int -> unit + method virtual flush : unit + end + + class factory = + object + method getTransport (t : t) = t + end + + class virtual server_t = + object (self) + method virtual listen : unit + method accept = self#acceptImpl + method virtual close : unit + method virtual acceptImpl : t + end + +end;; + + + +module Protocol = +struct + type t_type = + | T_STOP + | T_VOID + | T_BOOL + | T_BYTE + | T_I08 + | T_I16 + | T_I32 + | T_U64 + | T_I64 + | T_DOUBLE + | T_STRING + | T_UTF7 + | T_STRUCT + | T_MAP + | T_SET + | T_LIST + | T_UTF8 + | T_UTF16 + + let t_type_to_i = function + T_STOP -> 0 + | T_VOID -> 1 + | T_BOOL -> 2 + | T_BYTE -> 3 + | T_I08 -> 3 + | T_I16 -> 6 + | T_I32 -> 8 + | T_U64 -> 9 + | T_I64 -> 10 + | T_DOUBLE -> 4 + | T_STRING -> 11 + | T_UTF7 -> 11 + | T_STRUCT -> 12 + | T_MAP -> 13 + | T_SET -> 14 + | T_LIST -> 15 + | T_UTF8 -> 16 + | T_UTF16 -> 17 + + let t_type_of_i = function + 0 -> T_STOP + | 1 -> T_VOID + | 2 -> T_BOOL + | 3 -> T_BYTE + | 6-> T_I16 + | 8 -> T_I32 + | 9 -> T_U64 + | 10 -> T_I64 + | 4 -> T_DOUBLE + | 11 -> T_STRING + | 12 -> T_STRUCT + | 13 -> T_MAP + | 14 -> T_SET + | 15 -> T_LIST + | 16 -> T_UTF8 + | 17 -> T_UTF16 + | _ -> raise Thrift_error + + type message_type = + | CALL + | REPLY + | EXCEPTION + | ONEWAY + + let message_type_to_i = function + | CALL -> 1 + | REPLY -> 2 + | EXCEPTION -> 3 + | ONEWAY -> 4 + + let message_type_of_i = function + | 1 -> CALL + | 2 -> REPLY + | 3 -> EXCEPTION + | 4 -> ONEWAY + | _ -> raise Thrift_error + + class virtual t (trans: Transport.t) = + object (self) + val mutable trans_ = trans + method getTransport = trans_ + (* writing methods *) + method virtual writeMessageBegin : string * message_type * int -> unit + method virtual writeMessageEnd : unit + method virtual writeStructBegin : string -> unit + method virtual writeStructEnd : unit + method virtual writeFieldBegin : string * t_type * int -> unit + method virtual writeFieldEnd : unit + method virtual writeFieldStop : unit + method virtual writeMapBegin : t_type * t_type * int -> unit + method virtual writeMapEnd : unit + method virtual writeListBegin : t_type * int -> unit + method virtual writeListEnd : unit + method virtual writeSetBegin : t_type * int -> unit + method virtual writeSetEnd : unit + method virtual writeBool : bool -> unit + method virtual writeByte : int -> unit + method virtual writeI16 : int -> unit + method virtual writeI32 : int -> unit + method virtual writeI64 : Int64.t -> unit + method virtual writeDouble : float -> unit + method virtual writeString : string -> unit + method virtual writeBinary : string -> unit + (* reading methods *) + method virtual readMessageBegin : string * message_type * int + method virtual readMessageEnd : unit + method virtual readStructBegin : string + method virtual readStructEnd : unit + method virtual readFieldBegin : string * t_type * int + method virtual readFieldEnd : unit + method virtual readMapBegin : t_type * t_type * int + method virtual readMapEnd : unit + method virtual readListBegin : t_type * int + method virtual readListEnd : unit + method virtual readSetBegin : t_type * int + method virtual readSetEnd : unit + method virtual readBool : bool + method virtual readByte : int + method virtual readI16 : int + method virtual readI32: int + method virtual readI64 : Int64.t + method virtual readDouble : float + method virtual readString : string + method virtual readBinary : string + (* skippage *) + method skip typ = + match typ with + | T_STOP -> () + | T_VOID -> () + | T_BOOL -> ignore self#readBool + | T_BYTE + | T_I08 -> ignore self#readByte + | T_I16 -> ignore self#readI16 + | T_I32 -> ignore self#readI32 + | T_U64 + | T_I64 -> ignore self#readI64 + | T_DOUBLE -> ignore self#readDouble + | T_STRING -> ignore self#readString + | T_UTF7 -> () + | T_STRUCT -> ignore ((ignore self#readStructBegin); + (try + while true do + let (_,t,_) = self#readFieldBegin in + if t = T_STOP then + raise Break + else + (self#skip t; + self#readFieldEnd) + done + with Break -> ()); + self#readStructEnd) + | T_MAP -> ignore (let (k,v,s) = self#readMapBegin in + for i=0 to s do + self#skip k; + self#skip v; + done; + self#readMapEnd) + | T_SET -> ignore (let (t,s) = self#readSetBegin in + for i=0 to s do + self#skip t + done; + self#readSetEnd) + | T_LIST -> ignore (let (t,s) = self#readListBegin in + for i=0 to s do + self#skip t + done; + self#readListEnd) + | T_UTF8 -> () + | T_UTF16 -> () + end + + class virtual factory = + object + method virtual getProtocol : Transport.t -> t + end + + type exn_type = + | UNKNOWN + | INVALID_DATA + | NEGATIVE_SIZE + | SIZE_LIMIT + | BAD_VERSION + + exception E of exn_type * string;; + +end;; + + +module Processor = +struct + class virtual t = + object + method virtual process : Protocol.t -> Protocol.t -> bool + end;; + + class factory (processor : t) = + object + val processor_ = processor + method getProcessor (trans : Transport.t) = processor_ + end;; +end + + +(* Ugly *) +module Application_Exn = +struct + type typ= + | UNKNOWN + | UNKNOWN_METHOD + | INVALID_MESSAGE_TYPE + | WRONG_METHOD_NAME + | BAD_SEQUENCE_ID + | MISSING_RESULT + + let typ_of_i = function + 0 -> UNKNOWN + | 1 -> UNKNOWN_METHOD + | 2 -> INVALID_MESSAGE_TYPE + | 3 -> WRONG_METHOD_NAME + | 4 -> BAD_SEQUENCE_ID + | 5 -> MISSING_RESULT + | _ -> raise Thrift_error;; + let typ_to_i = function + | UNKNOWN -> 0 + | UNKNOWN_METHOD -> 1 + | INVALID_MESSAGE_TYPE -> 2 + | WRONG_METHOD_NAME -> 3 + | BAD_SEQUENCE_ID -> 4 + | MISSING_RESULT -> 5 + + class t = + object (self) + inherit t_exn + val mutable typ = UNKNOWN + method get_type = typ + method set_type t = typ <- t + method write (oprot : Protocol.t) = + oprot#writeStructBegin "TApplicationExeception"; + if self#get_message != "" then + (oprot#writeFieldBegin ("message",Protocol.T_STRING, 1); + oprot#writeString self#get_message; + oprot#writeFieldEnd) + else (); + oprot#writeFieldBegin ("type",Protocol.T_I32,2); + oprot#writeI32 (typ_to_i typ); + oprot#writeFieldEnd; + oprot#writeFieldStop; + oprot#writeStructEnd + end;; + + let create typ msg = + let e = new t in + e#set_type typ; + e#set_message msg; + e + + let read (iprot : Protocol.t) = + let msg = ref "" in + let typ = ref 0 in + ignore iprot#readStructBegin; + (try + while true do + let (name,ft,id) =iprot#readFieldBegin in + if ft = Protocol.T_STOP then + raise Break + else (); + (match id with + | 1 -> (if ft = Protocol.T_STRING then + msg := (iprot#readString) + else + iprot#skip ft) + | 2 -> (if ft = Protocol.T_I32 then + typ := iprot#readI32 + else + iprot#skip ft) + | _ -> iprot#skip ft); + iprot#readFieldEnd + done + with Break -> ()); + iprot#readStructEnd; + let e = new t in + e#set_type (typ_of_i !typ); + e#set_message !msg; + e;; + + exception E of t +end;; diff --git a/lib/perl/Makefile.PL b/lib/perl/Makefile.PL new file mode 100644 index 00000000..94ea37ce --- /dev/null +++ b/lib/perl/Makefile.PL @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +use ExtUtils::MakeMaker; +WriteMakefile( 'NAME' => 'Thrift', + 'VERSION_FROM' => 'lib/Thrift.pm', + 'PREREQ_PM' => { + 'Bit::Vector' => 0, + 'Class::Accessor' => 0 + }, + ($] >= 5.005 ? + ( AUTHOR => 'T Jake Luciani ') : ()), + ); diff --git a/lib/perl/Makefile.am b/lib/perl/Makefile.am new file mode 100644 index 00000000..163d0158 --- /dev/null +++ b/lib/perl/Makefile.am @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +SUBDIRS = test + +Makefile-perl.mk : Makefile.PL + $(PERL) Makefile.PL MAKEFILE=Makefile-perl.mk INSTALLDIRS=$(INSTALLDIRS) + +all-local: Makefile-perl.mk + $(MAKE) -f Makefile-perl.mk + find blib -name 'Makefile*' -exec rm -f {} \; + +check-local: + $(PERL) -Iblib/lib -I@abs_srcdir@ -I@builddir@/test/gen-perl \ + @abs_srcdir@/test.pl @abs_srcdir@/test/*.t + +install-exec-local: Makefile-perl.mk + $(MAKE) -f Makefile-perl.mk install DESTDIR=$(DESTDIR)/ + +clean-local: + if test -f Makefile-perl.mk ; then \ + $(MAKE) -f Makefile-perl.mk clean ; \ + fi + rm -f Makefile-perl.mk.old + +EXTRA_DIST = \ + Makefile.PL \ + test.pl \ + lib/Thrift.pm \ + lib/Thrift.pm \ + lib/Thrift/BinaryProtocol.pm \ + lib/Thrift/BufferedTransport.pm \ + lib/Thrift/FramedTransport.pm \ + lib/Thrift/HttpClient.pm \ + lib/Thrift/MemoryBuffer.pm \ + lib/Thrift/Protocol.pm \ + lib/Thrift/Socket.pm \ + lib/Thrift/Transport.pm diff --git a/lib/perl/README b/lib/perl/README new file mode 100644 index 00000000..691488b4 --- /dev/null +++ b/lib/perl/README @@ -0,0 +1,41 @@ +Thrift Perl Software Library + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Using Thrift with Perl +===================== + +Thrift requires Perl >= 5.6.0 + +Exceptions are thrown with die so be sure to wrap eval{} statments +around any code that contains exceptions. + +The 64bit Integers work only upto 2^42 on my machine :-? +Math::BigInt is probably needed. + +Please see tutoral and test dirs for examples... + +Dependencies +============ + +Bit::Vector - comes with modern perl installations. +Class::Accessor + diff --git a/lib/perl/lib/Thrift.pm b/lib/perl/lib/Thrift.pm new file mode 100644 index 00000000..fe0f8e72 --- /dev/null +++ b/lib/perl/lib/Thrift.pm @@ -0,0 +1,177 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +our $VERSION = '0.1'; + +require 5.6.0; +use strict; +use warnings; + +# +# Data types that can be sent via Thrift +# +package TType; +use constant STOP => 0; +use constant VOID => 1; +use constant BOOL => 2; +use constant BYTE => 3; +use constant I08 => 3; +use constant DOUBLE => 4; +use constant I16 => 6; +use constant I32 => 8; +use constant I64 => 10; +use constant STRING => 11; +use constant UTF7 => 11; +use constant STRUCT => 12; +use constant MAP => 13; +use constant SET => 14; +use constant LIST => 15; +use constant UTF8 => 16; +use constant UTF16 => 17; +1; + +# +# Message types for RPC +# +package TMessageType; +use constant CALL => 1; +use constant REPLY => 2; +use constant EXCEPTION => 3; +use constant ONEWAY => 4; +1; + +package Thrift::TException; + +sub new { + my $classname = shift; + my $self = {message => shift, code => shift || 0}; + + return bless($self,$classname); +} +1; + +package TApplicationException; +use base('Thrift::TException'); + +use constant UNKNOWN => 0; +use constant UNKNOWN_METHOD => 1; +use constant INVALID_MESSAGE_TYPE => 2; +use constant WRONG_METHOD_NAME => 3; +use constant BAD_SEQUENCE_ID => 4; +use constant MISSING_RESULT => 5; + +sub new { + my $classname = shift; + + my $self = $classname->SUPER::new(); + + return bless($self,$classname); +} + +sub read { + my $self = shift; + my $input = shift; + + my $xfer = 0; + my $fname = undef; + my $ftype = 0; + my $fid = 0; + + $xfer += $input->readStructBegin($fname); + + while (1) + { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + last; next; + } + + SWITCH: for($fid) + { + /1/ && do{ + + if ($ftype == TType::STRING) { + $xfer += $input->readString($self->{message}); + } else { + $xfer += $input->skip($ftype); + } + + last; + }; + + /2/ && do{ + if ($ftype == TType::I32) { + $xfer += $input->readI32($self->{code}); + } else { + $xfer += $input->skip($ftype); + } + last; + }; + + $xfer += $input->skip($ftype); + } + + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + + return $xfer; +} + +sub write { + my $self = shift; + my $output = shift; + + my $xfer = 0; + + $xfer += $output->writeStructBegin('TApplicationException'); + + if ($self->getMessage()) { + $xfer += $output->writeFieldBegin('message', TType::STRING, 1); + $xfer += $output->writeString($self->getMessage()); + $xfer += $output->writeFieldEnd(); + } + + if ($self->getCode()) { + $xfer += $output->writeFieldBegin('type', TType::I32, 2); + $xfer += $output->writeI32($self->getCode()); + $xfer += $output->writeFieldEnd(); + } + + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + + return $xfer; +} + +sub getMessage +{ + my $self = shift; + + return $self->{message}; +} + +sub getCode +{ + my $self = shift; + + return $self->{code}; +} + +1; diff --git a/lib/perl/lib/Thrift/BinaryProtocol.pm b/lib/perl/lib/Thrift/BinaryProtocol.pm new file mode 100644 index 00000000..0e5d61d3 --- /dev/null +++ b/lib/perl/lib/Thrift/BinaryProtocol.pm @@ -0,0 +1,498 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 5.6.0; + +use strict; +use warnings; + +use utf8; +use Encode; + +use Thrift; +use Thrift::Protocol; + +use Bit::Vector; + +# +# Binary implementation of the Thrift protocol. +# +package Thrift::BinaryProtocol; +use base('Thrift::Protocol'); + +use constant VERSION_MASK => 0xffff0000; +use constant VERSION_1 => 0x80010000; + +sub new +{ + my $classname = shift; + my $trans = shift; + my $self = $classname->SUPER::new($trans); + + return bless($self,$classname); +} + +sub writeMessageBegin +{ + my $self = shift; + my ($name, $type, $seqid) = @_; + + return + $self->writeI32(VERSION_1 | $type) + + $self->writeString($name) + + $self->writeI32($seqid); +} + +sub writeMessageEnd +{ + my $self = shift; + return 0; +} + +sub writeStructBegin{ + my $self = shift; + my $name = shift; + return 0; +} + +sub writeStructEnd +{ + my $self = shift; + return 0; +} + +sub writeFieldBegin +{ + my $self = shift; + my ($fieldName, $fieldType, $fieldId) = @_; + + return + $self->writeByte($fieldType) + + $self->writeI16($fieldId); +} + +sub writeFieldEnd +{ + my $self = shift; + return 0; +} + +sub writeFieldStop +{ + my $self = shift; + return $self->writeByte(TType::STOP); +} + +sub writeMapBegin +{ + my $self = shift; + my ($keyType, $valType, $size) = @_; + + return + $self->writeByte($keyType) + + $self->writeByte($valType) + + $self->writeI32($size); +} + +sub writeMapEnd +{ + my $self = shift; + return 0; +} + +sub writeListBegin +{ + my $self = shift; + my ($elemType, $size) = @_; + + return + $self->writeByte($elemType) + + $self->writeI32($size); +} + +sub writeListEnd +{ + my $self = shift; + return 0; +} + +sub writeSetBegin +{ + my $self = shift; + my ($elemType, $size) = @_; + + return + $self->writeByte($elemType) + + $self->writeI32($size); +} + +sub writeSetEnd +{ + my $self = shift; + return 0; +} + +sub writeBool +{ + my $self = shift; + my $value = shift; + + my $data = pack('c', $value ? 1 : 0); + $self->{trans}->write($data, 1); + return 1; +} + +sub writeByte +{ + my $self = shift; + my $value= shift; + + my $data = pack('c', $value); + $self->{trans}->write($data, 1); + return 1; +} + +sub writeI16 +{ + my $self = shift; + my $value= shift; + + my $data = pack('n', $value); + $self->{trans}->write($data, 2); + return 2; +} + +sub writeI32 +{ + my $self = shift; + my $value= shift; + + my $data = pack('N', $value); + $self->{trans}->write($data, 4); + return 4; +} + +sub writeI64 +{ + my $self = shift; + my $value= shift; + my $data; + + my $vec; + #stop annoying error + $vec = Bit::Vector->new_Dec(64, $value); + $data = pack 'NN', $vec->Chunk_Read(32, 32), $vec->Chunk_Read(32, 0); + + $self->{trans}->write($data, 8); + + return 8; +} + + +sub writeDouble +{ + my $self = shift; + my $value= shift; + + my $data = pack('d', $value); + $self->{trans}->write(scalar reverse($data), 8); + return 8; +} + +sub writeString{ + my $self = shift; + my $value= shift; + + if( utf8::is_utf8($value) ){ + $value = Encode::encode_utf8($value); + } + + my $len = length($value); + + my $result = $self->writeI32($len); + + if ($len) { + $self->{trans}->write($value,$len); + } + return $result + $len; + } + + +# +#All references +# +sub readMessageBegin +{ + my $self = shift; + my ($name, $type, $seqid) = @_; + + my $version = 0; + my $result = $self->readI32(\$version); + if (($version & VERSION_MASK) > 0) { + if (($version & VERSION_MASK) != VERSION_1) { + die new Thrift::TException('Missing version identifier') + } + $$type = $version & 0x000000ff; + return + $result + + $self->readString($name) + + $self->readI32($seqid); + } else { # old client support code + return + $result + + $self->readStringBody($name, $version) + # version here holds the size of the string + $self->readByte($type) + + $self->readI32($seqid); + } +} + +sub readMessageEnd +{ + my $self = shift; + return 0; +} + +sub readStructBegin +{ + my $self = shift; + my $name = shift; + + $$name = ''; + + return 0; +} + +sub readStructEnd +{ + my $self = shift; + return 0; +} + +sub readFieldBegin +{ + my $self = shift; + my ($name, $fieldType, $fieldId) = @_; + + my $result = $self->readByte($fieldType); + + if ($$fieldType == TType::STOP) { + $$fieldId = 0; + return $result; + } + + $result += $self->readI16($fieldId); + + return $result; +} + +sub readFieldEnd() { + my $self = shift; + return 0; +} + +sub readMapBegin +{ + my $self = shift; + my ($keyType, $valType, $size) = @_; + + return + $self->readByte($keyType) + + $self->readByte($valType) + + $self->readI32($size); +} + +sub readMapEnd() +{ + my $self = shift; + return 0; +} + +sub readListBegin +{ + my $self = shift; + my ($elemType, $size) = @_; + + return + $self->readByte($elemType) + + $self->readI32($size); +} + +sub readListEnd +{ + my $self = shift; + return 0; +} + +sub readSetBegin +{ + my $self = shift; + my ($elemType, $size) = @_; + + return + $self->readByte($elemType) + + $self->readI32($size); +} + +sub readSetEnd +{ + my $self = shift; + return 0; +} + +sub readBool +{ + my $self = shift; + my $value = shift; + + my $data = $self->{trans}->readAll(1); + my @arr = unpack('c', $data); + $$value = $arr[0] == 1; + return 1; +} + +sub readByte +{ + my $self = shift; + my $value = shift; + + my $data = $self->{trans}->readAll(1); + my @arr = unpack('c', $data); + $$value = $arr[0]; + return 1; +} + +sub readI16 +{ + my $self = shift; + my $value = shift; + + my $data = $self->{trans}->readAll(2); + + my @arr = unpack('n', $data); + + $$value = $arr[0]; + + if ($$value > 0x7fff) { + $$value = 0 - (($$value - 1) ^ 0xffff); + } + + return 2; +} + +sub readI32 +{ + my $self = shift; + my $value= shift; + + my $data = $self->{trans}->readAll(4); + my @arr = unpack('N', $data); + + $$value = $arr[0]; + if ($$value > 0x7fffffff) { + $$value = 0 - (($$value - 1) ^ 0xffffffff); + } + return 4; +} + +sub readI64 +{ + my $self = shift; + my $value = shift; + + my $data = $self->{trans}->readAll(8); + + my ($hi,$lo)=unpack('NN',$data); + + my $vec = new Bit::Vector(64); + + $vec->Chunk_Store(32,32,$hi); + $vec->Chunk_Store(32,0,$lo); + + $$value = $vec->to_Dec(); + + return 8; +} + +sub readDouble +{ + my $self = shift; + my $value = shift; + + my $data = scalar reverse($self->{trans}->readAll(8)); + my @arr = unpack('d', $data); + + $$value = $arr[0]; + + return 8; +} + +sub readString +{ + my $self = shift; + my $value = shift; + + my $len; + my $result = $self->readI32(\$len); + + if ($len) { + $$value = $self->{trans}->readAll($len); + } else { + $$value = ''; + } + + return $result + $len; +} + +sub readStringBody +{ + my $self = shift; + my $value = shift; + my $len = shift; + + if ($len) { + $$value = $self->{trans}->readAll($len); + } else { + $$value = ''; + } + + return $len; +} + +# +# Binary Protocol Factory +# +package TBinaryProtocolFactory; +use base('TProtocolFactory'); + +sub new +{ + my $classname = shift; + my $self = $classname->SUPER::new(); + + return bless($self,$classname); +} + +sub getProtocol{ + my $self = shift; + my $trans = shift; + + return new TBinaryProtocol($trans); +} + +1; diff --git a/lib/perl/lib/Thrift/BufferedTransport.pm b/lib/perl/lib/Thrift/BufferedTransport.pm new file mode 100644 index 00000000..bef564d6 --- /dev/null +++ b/lib/perl/lib/Thrift/BufferedTransport.pm @@ -0,0 +1,109 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 5.6.0; +use strict; +use warnings; + +use Thrift; +use Thrift::Transport; + +package Thrift::BufferedTransport; +use base('Thrift::Transport'); + +sub new +{ + my $classname = shift; + my $transport = shift; + my $rBufSize = shift || 512; + my $wBufSize = shift || 512; + + my $self = { + transport => $transport, + rBufSize => $rBufSize, + wBufSize => $wBufSize, + wBuf => '', + rBuf => '', + }; + + return bless($self,$classname); +} + +sub isOpen +{ + my $self = shift; + + return $self->{transport}->isOpen(); +} + +sub open +{ + my $self = shift; + $self->{transport}->open(); +} + +sub close() +{ + my $self = shift; + $self->{transport}->close(); +} + +sub readAll +{ + my $self = shift; + my $len = shift; + + return $self->{transport}->readAll($len); +} + +sub read +{ + my $self = shift; + my $len = shift; + my $ret; + + # Methinks Perl is already buffering these for us + return $self->{transport}->read($len); +} + +sub write +{ + my $self = shift; + my $buf = shift; + + $self->{wBuf} .= $buf; + if (length($self->{wBuf}) >= $self->{wBufSize}) { + $self->{transport}->write($self->{wBuf}); + $self->{wBuf} = ''; + } +} + +sub flush +{ + my $self = shift; + + if (length($self->{wBuf}) > 0) { + $self->{transport}->write($self->{wBuf}); + $self->{wBuf} = ''; + } + $self->{transport}->flush(); +} + + +1; diff --git a/lib/perl/lib/Thrift/FramedTransport.pm b/lib/perl/lib/Thrift/FramedTransport.pm new file mode 100644 index 00000000..b78b1989 --- /dev/null +++ b/lib/perl/lib/Thrift/FramedTransport.pm @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +use strict; +use warnings; + +use Thrift; +use Thrift::Transport; + +# +# Framed transport. Writes and reads data in chunks that are stamped with +# their length. +# +# @package thrift.transport +# +package Thrift::FramedTransport; + +use base('Thrift::Transport'); + +sub new +{ + my $classname = shift; + my $transport = shift; + my $read = shift || 1; + my $write = shift || 1; + + my $self = { + transport => $transport, + read => $read, + write => $write, + wBuf => '', + rBuf => '', + }; + + return bless($self,$classname); +} + +sub isOpen +{ + my $self = shift; + return $self->{transport}->isOpen(); +} + +sub open +{ + my $self = shift; + + $self->{transport}->open(); +} + +sub close +{ + my $self = shift; + + $self->{transport}->close(); +} + +# +# Reads from the buffer. When more data is required reads another entire +# chunk and serves future reads out of that. +# +# @param int $len How much data +# +sub read +{ + + my $self = shift; + my $len = shift; + + if (!$self->{read}) { + return $self->{transport}->read($len); + } + + if (length($self->{rBuf}) == 0) { + $self->_readFrame(); + } + + + # Just return full buff + if ($len > length($self->{rBuf})) { + my $out = $self->{rBuf}; + $self->{rBuf} = ''; + return $out; + } + + # Return substr + my $out = substr($self->{rBuf}, 0, $len); + $self->{rBuf} = substr($self->{rBuf}, $len); + return $out; +} + +# +# Reads a chunk of data into the internal read buffer. +# (private) +sub _readFrame +{ + my $self = shift; + my $buf = $self->{transport}->readAll(4); + my @val = unpack('N', $buf); + my $sz = $val[0]; + + $self->{rBuf} = $self->{transport}->readAll($sz); +} + +# +# Writes some data to the pending output buffer. +# +# @param string $buf The data +# @param int $len Limit of bytes to write +# +sub write +{ + my $self = shift; + my $buf = shift; + my $len = shift; + + unless($self->{write}) { + return $self->{transport}->write($buf, $len); + } + + if ( defined $len && $len < length($buf)) { + $buf = substr($buf, 0, $len); + } + + $self->{wBuf} .= $buf; + } + +# +# Writes the output buffer to the stream in the format of a 4-byte length +# followed by the actual data. +# +sub flush +{ + my $self = shift; + + unless ($self->{write}) { + return $self->{transport}->flush(); + } + + my $out = pack('N', length($self->{wBuf})); + $out .= $self->{wBuf}; + $self->{transport}->write($out); + $self->{transport}->flush(); + $self->{wBuf} = ''; + +} + +1; diff --git a/lib/perl/lib/Thrift/HttpClient.pm b/lib/perl/lib/Thrift/HttpClient.pm new file mode 100644 index 00000000..d6fc8be3 --- /dev/null +++ b/lib/perl/lib/Thrift/HttpClient.pm @@ -0,0 +1,200 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 5.6.0; +use strict; +use warnings; + +use Thrift; +use Thrift::Transport; + +use HTTP::Request; +use LWP::UserAgent; +use IO::String; + +package Thrift::HttpClient; + +use base('Thrift::Transport'); + +sub new +{ + my $classname = shift; + my $url = shift || 'http://localhost:9090'; + my $debugHandler = shift; + + my $out = IO::String->new; + binmode($out); + + my $self = { + url => $url, + out => $out, + debugHandler => $debugHandler, + debug => 0, + sendTimeout => 100, + recvTimeout => 750, + handle => undef, + }; + + return bless($self,$classname); +} + +sub setSendTimeout +{ + my $self = shift; + my $timeout = shift; + + $self->{sendTimeout} = $timeout; +} + +sub setRecvTimeout +{ + my $self = shift; + my $timeout = shift; + + $self->{recvTimeout} = $timeout; +} + + +# +#Sets debugging output on or off +# +# @param bool $debug +# +sub setDebug +{ + my $self = shift; + my $debug = shift; + + $self->{debug} = $debug; +} + +# +# Tests whether this is open +# +# @return bool true if the socket is open +# +sub isOpen +{ + return 1; +} + +sub open {} + +# +# Cleans up the buffer. +# +sub close +{ + my $self = shift; + if (defined($self->{io})) { + close($self->{io}); + $self->{io} = undef; + } +} + +# +# Guarantees that the full amount of data is read. +# +# @return string The data, of exact length +# @throws TTransportException if cannot read data +# +sub readAll +{ + my $self = shift; + my $len = shift; + + my $buf = $self->read($len); + + if (!defined($buf)) { + die new Thrift::TException('TSocket: Could not read '.$len.' bytes from input buffer'); + } + return $buf; +} + +# +# Read and return string +# +sub read +{ + my $self = shift; + my $len = shift; + + my $buf; + + my $in = $self->{in}; + + if (!defined($in)) { + die new Thrift::TException("Response buffer is empty, no request."); + } + eval { + my $ret = sysread($in, $buf, $len); + if (! defined($ret)) { + die new Thrift::TException("No more data available."); + } + }; if($@){ + die new Thrift::TException($@); + } + + return $buf; +} + +# +# Write string +# +sub write +{ + my $self = shift; + my $buf = shift; + $self->{out}->print($buf); +} + +# +# Flush output (do the actual HTTP/HTTPS request) +# +sub flush +{ + my $self = shift; + + my $ua = LWP::UserAgent->new('timeout' => ($self->{sendTimeout} / 1000), + 'agent' => 'Perl/THttpClient' + ); + $ua->default_header('Accept' => 'application/x-thrift'); + $ua->default_header('Content-Type' => 'application/x-thrift'); + $ua->cookie_jar({}); # hash to remember cookies between redirects + + my $out = $self->{out}; + $out->setpos(0); # rewind + my $buf = join('', <$out>); + + my $request = new HTTP::Request(POST => $self->{url}, undef, $buf); + my $response = $ua->request($request); + my $content_ref = $response->content_ref; + + my $in = IO::String->new($content_ref); + binmode($in); + $self->{in} = $in; + $in->setpos(0); # rewind + + # reset write buffer + $out = IO::String->new; + binmode($out); + $self->{out} = $out; +} + +1; diff --git a/lib/perl/lib/Thrift/MemoryBuffer.pm b/lib/perl/lib/Thrift/MemoryBuffer.pm new file mode 100644 index 00000000..32f14424 --- /dev/null +++ b/lib/perl/lib/Thrift/MemoryBuffer.pm @@ -0,0 +1,126 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 5.6.0; +use strict; +use warnings; + +use Thrift; +use Thrift::Transport; + +package Thrift::MemoryBuffer; +use base('Thrift::Transport'); + +sub new +{ + my $classname = shift; + + my $bufferSize= shift || 1024; + + my $self = { + buffer => '', + bufferSize=> $bufferSize, + wPos => 0, + rPos => 0, + }; + + return bless($self,$classname); +} + +sub isOpen +{ + return 1; +} + +sub open +{ + +} + +sub close +{ + +} + +sub peek +{ + my $self = shift; + return($self->{rPos} < $self->{wPos}); +} + + +sub getBuffer +{ + my $self = shift; + return $self->{buffer}; +} + +sub resetBuffer +{ + my $self = shift; + + my $new_buffer = shift || ''; + + $self->{buffer} = $new_buffer; + $self->{bufferSize} = length($new_buffer); + $self->{wPos} = length($new_buffer); + $self->{rPos} = 0; +} + +sub available +{ + my $self = shift; + return ($self->{wPos} - $self->{rPos}); +} + +sub read +{ + my $self = shift; + my $len = shift; + my $ret; + + my $avail = ($self->{wPos} - $self->{rPos}); + return '' if $avail == 0; + + #how much to give + my $give = $len; + $give = $avail if $avail < $len; + + $ret = substr($self->{buffer},$self->{rPos},$give); + + $self->{rPos} += $give; + + return $ret; +} + +sub write +{ + my $self = shift; + my $buf = shift; + + $self->{buffer} .= $buf; + $self->{wPos} += length($buf); +} + +sub flush +{ + +} + +1; diff --git a/lib/perl/lib/Thrift/Protocol.pm b/lib/perl/lib/Thrift/Protocol.pm new file mode 100644 index 00000000..034711f3 --- /dev/null +++ b/lib/perl/lib/Thrift/Protocol.pm @@ -0,0 +1,543 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 5.6.0; +use strict; +use warnings; + +use Thrift; + +# +# Protocol exceptions +# +package TProtocolException; +use base('Thrift::TException'); + +use constant UNKNOWN => 0; +use constant INVALID_DATA => 1; +use constant NEGATIVE_SIZE => 2; +use constant SIZE_LIMIT => 3; +use constant BAD_VERSION => 4; + +sub new { + my $classname = shift; + + my $self = $classname->SUPER::new(); + + return bless($self,$classname); +} + +# +# Protocol base class module. +# +package Thrift::Protocol; + +sub new { + my $classname = shift; + my $self = {}; + + my $trans = shift; + $self->{trans}= $trans; + + return bless($self,$classname); +} + +sub getTransport +{ + my $self = shift; + + return $self->{trans}; +} + +# +# Writes the message header +# +# @param string $name Function name +# @param int $type message type TMessageType::CALL or TMessageType::REPLY +# @param int $seqid The sequence id of this message +# +sub writeMessageBegin +{ + my ($name, $type, $seqid); + die "abstract"; +} + +# +# Close the message +# +sub writeMessageEnd { + die "abstract"; +} + +# +# Writes a struct header. +# +# @param string $name Struct name +# @throws TException on write error +# @return int How many bytes written +# +sub writeStructBegin { + my ($name); + + die "abstract"; +} + +# +# Close a struct. +# +# @throws TException on write error +# @return int How many bytes written +# +sub writeStructEnd { + die "abstract"; +} + +# +# Starts a field. +# +# @param string $name Field name +# @param int $type Field type +# @param int $fid Field id +# @throws TException on write error +# @return int How many bytes written +# +sub writeFieldBegin { + my ($fieldName, $fieldType, $fieldId); + + die "abstract"; +} + +sub writeFieldEnd { + die "abstract"; +} + +sub writeFieldStop { + die "abstract"; +} + +sub writeMapBegin { + my ($keyType, $valType, $size); + + die "abstract"; +} + +sub writeMapEnd { + die "abstract"; +} + +sub writeListBegin { + my ($elemType, $size); + die "abstract"; +} + +sub writeListEnd { + die "abstract"; +} + +sub writeSetBegin { + my ($elemType, $size); + die "abstract"; +} + +sub writeSetEnd { + die "abstract"; +} + +sub writeBool { + my ($bool); + die "abstract"; +} + +sub writeByte { + my ($byte); + die "abstract"; +} + +sub writeI16 { + my ($i16); + die "abstract"; +} + +sub writeI32 { + my ($i32); + die "abstract"; +} + +sub writeI64 { + my ($i64); + die "abstract"; +} + +sub writeDouble { + my ($dub); + die "abstract"; +} + +sub writeString +{ + my ($str); + die "abstract"; +} + +# +# Reads the message header +# +# @param string $name Function name +# @param int $type message type TMessageType::CALL or TMessageType::REPLY +# @parem int $seqid The sequence id of this message +# +sub readMessageBegin +{ + my ($name, $type, $seqid); + die "abstract"; +} + +# +# Read the close of message +# +sub readMessageEnd +{ + die "abstract"; +} + +sub readStructBegin +{ + my($name); + + die "abstract"; +} + +sub readStructEnd +{ + die "abstract"; +} + +sub readFieldBegin +{ + my ($name, $fieldType, $fieldId); + die "abstract"; +} + +sub readFieldEnd +{ + die "abstract"; +} + +sub readMapBegin +{ + my ($keyType, $valType, $size); + die "abstract"; +} + +sub readMapEnd +{ + die "abstract"; +} + +sub readListBegin +{ + my ($elemType, $size); + die "abstract"; +} + +sub readListEnd +{ + die "abstract"; +} + +sub readSetBegin +{ + my ($elemType, $size); + die "abstract"; +} + +sub readSetEnd +{ + die "abstract"; +} + +sub readBool +{ + my ($bool); + die "abstract"; +} + +sub readByte +{ + my ($byte); + die "abstract"; +} + +sub readI16 +{ + my ($i16); + die "abstract"; +} + +sub readI32 +{ + my ($i32); + die "abstract"; +} + +sub readI64 +{ + my ($i64); + die "abstract"; +} + +sub readDouble +{ + my ($dub); + die "abstract"; +} + +sub readString +{ + my ($str); + die "abstract"; +} + +# +# The skip function is a utility to parse over unrecognized data without +# causing corruption. +# +# @param TType $type What type is it +# +sub skip +{ + my $self = shift; + my $type = shift; + + my $ref; + my $result; + my $i; + + if($type == TType::BOOL) + { + return $self->readBool(\$ref); + } + elsif($type == TType::BYTE){ + return $self->readByte(\$ref); + } + elsif($type == TType::I16){ + return $self->readI16(\$ref); + } + elsif($type == TType::I32){ + return $self->readI32(\$ref); + } + elsif($type == TType::I64){ + return $self->readI64(\$ref); + } + elsif($type == TType::DOUBLE){ + return $self->readDouble(\$ref); + } + elsif($type == TType::STRING) + { + return $self->readString(\$ref); + } + elsif($type == TType::STRUCT) + { + $result = $self->readStructBegin(\$ref); + while (1) { + my ($ftype,$fid); + $result += $self->readFieldBegin(\$ref, \$ftype, \$fid); + if ($ftype == TType::STOP) { + last; + } + $result += $self->skip($ftype); + $result += $self->readFieldEnd(); + } + $result += $self->readStructEnd(); + return $result; + } + elsif($type == TType::MAP) + { + my($keyType,$valType,$size); + $result = $self->readMapBegin(\$keyType, \$valType, \$size); + for ($i = 0; $i < $size; $i++) { + $result += $self->skip($keyType); + $result += $self->skip($valType); + } + $result += $self->readMapEnd(); + return $result; + } + elsif($type == TType::SET) + { + my ($elemType,$size); + $result = $self->readSetBegin(\$elemType, \$size); + for ($i = 0; $i < $size; $i++) { + $result += $self->skip($elemType); + } + $result += $self->readSetEnd(); + return $result; + } + elsif($type == TType::LIST) + { + my ($elemType,$size); + $result = $self->readListBegin(\$elemType, \$size); + for ($i = 0; $i < $size; $i++) { + $result += $self->skip($elemType); + } + $result += $self->readListEnd(); + return $result; + } + + + return 0; + + } + +# +# Utility for skipping binary data +# +# @param TTransport $itrans TTransport object +# @param int $type Field type +# +sub skipBinary +{ + my $self = shift; + my $itrans = shift; + my $type = shift; + + if($type == TType::BOOL) + { + return $itrans->readAll(1); + } + elsif($type == TType::BYTE) + { + return $itrans->readAll(1); + } + elsif($type == TType::I16) + { + return $itrans->readAll(2); + } + elsif($type == TType::I32) + { + return $itrans->readAll(4); + } + elsif($type == TType::I64) + { + return $itrans->readAll(8); + } + elsif($type == TType::DOUBLE) + { + return $itrans->readAll(8); + } + elsif( $type == TType::STRING ) + { + my @len = unpack('N', $itrans->readAll(4)); + my $len = $len[0]; + if ($len > 0x7fffffff) { + $len = 0 - (($len - 1) ^ 0xffffffff); + } + return 4 + $itrans->readAll($len); + } + elsif( $type == TType::STRUCT ) + { + my $result = 0; + while (1) { + my $ftype = 0; + my $fid = 0; + my $data = $itrans->readAll(1); + my @arr = unpack('c', $data); + $ftype = $arr[0]; + if ($ftype == TType::STOP) { + last; + } + # I16 field id + $result += $itrans->readAll(2); + $result += $self->skipBinary($itrans, $ftype); + } + return $result; + } + elsif($type == TType::MAP) + { + # Ktype + my $data = $itrans->readAll(1); + my @arr = unpack('c', $data); + my $ktype = $arr[0]; + # Vtype + $data = $itrans->readAll(1); + @arr = unpack('c', $data); + my $vtype = $arr[0]; + # Size + $data = $itrans->readAll(4); + @arr = unpack('N', $data); + my $size = $arr[0]; + if ($size > 0x7fffffff) { + $size = 0 - (($size - 1) ^ 0xffffffff); + } + my $result = 6; + for (my $i = 0; $i < $size; $i++) { + $result += $self->skipBinary($itrans, $ktype); + $result += $self->skipBinary($itrans, $vtype); + } + return $result; + } + elsif($type == TType::SET || $type == TType::LIST) + { + # Vtype + my $data = $itrans->readAll(1); + my @arr = unpack('c', $data); + my $vtype = $arr[0]; + # Size + $data = $itrans->readAll(4); + @arr = unpack('N', $data); + my $size = $arr[0]; + if ($size > 0x7fffffff) { + $size = 0 - (($size - 1) ^ 0xffffffff); + } + my $result = 5; + for (my $i = 0; $i < $size; $i++) { + $result += $self->skipBinary($itrans, $vtype); + } + return $result; + } + + return 0; + +} + +# +# Protocol factory creates protocol objects from transports +# +package TProtocolFactory; + + +sub new { + my $classname = shift; + my $self = {}; + + return bless($self,$classname); +} + +# +# Build a protocol from the base transport +# +# @return TProtcol protocol +# +sub getProtocol +{ + my ($trans); + die "interface"; +} + + +1; diff --git a/lib/perl/lib/Thrift/Socket.pm b/lib/perl/lib/Thrift/Socket.pm new file mode 100644 index 00000000..67faa510 --- /dev/null +++ b/lib/perl/lib/Thrift/Socket.pm @@ -0,0 +1,271 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 5.6.0; +use strict; +use warnings; + +use Thrift; +use Thrift::Transport; + +use IO::Socket::INET; +use IO::Select; + +package Thrift::Socket; + +use base('Thrift::Transport'); + +sub new +{ + my $classname = shift; + my $host = shift || "localhost"; + my $port = shift || 9090; + my $debugHandler = shift; + + my $self = { + host => $host, + port => $port, + debugHandler => $debugHandler, + debug => 0, + sendTimeout => 100, + recvTimeout => 750, + handle => undef, + }; + + return bless($self,$classname); +} + + +sub setSendTimeout +{ + my $self = shift; + my $timeout = shift; + + $self->{sendTimeout} = $timeout; +} + +sub setRecvTimeout +{ + my $self = shift; + my $timeout = shift; + + $self->{recvTimeout} = $timeout; +} + + +# +#Sets debugging output on or off +# +# @param bool $debug +# +sub setDebug +{ + my $self = shift; + my $debug = shift; + + $self->{debug} = $debug; +} + +# +# Tests whether this is open +# +# @return bool true if the socket is open +# +sub isOpen +{ + my $self = shift; + + if( defined $self->{handle} ){ + return ($self->{handle}->handles())[0]->connected; + } + + return 0; +} + +# +# Connects the socket. +# +sub open +{ + my $self = shift; + + my $sock = IO::Socket::INET->new(PeerAddr => $self->{host}, + PeerPort => $self->{port}, + Proto => 'tcp', + Timeout => $self->{sendTimeout}/1000) + || do { + my $error = 'TSocket: Could not connect to '.$self->{host}.':'.$self->{port}.' ('.$!.')'; + + if ($self->{debug}) { + $self->{debugHandler}->($error); + } + + die new Thrift::TException($error); + + }; + + + $self->{handle} = new IO::Select( $sock ); +} + +# +# Closes the socket. +# +sub close +{ + my $self = shift; + + if( defined $self->{handle} ){ + close( ($self->{handle}->handles())[0] ); + } +} + +# +# Uses stream get contents to do the reading +# +# @param int $len How many bytes +# @return string Binary data +# +sub readAll +{ + my $self = shift; + my $len = shift; + + + return unless defined $self->{handle}; + + my $pre = ""; + while (1) { + + #check for timeout + my @sockets = $self->{handle}->can_read( $self->{recvTimeout} / 1000 ); + + if(@sockets == 0){ + die new Thrift::TException('TSocket: timed out reading '.$len.' bytes from '. + $self->{host}.':'.$self->{port}); + } + + my $sock = $sockets[0]; + + my ($buf,$sz); + $sock->recv($buf, $len); + + if (!defined $buf || $buf eq '') { + + die new Thrift::TException('TSocket: Could not read '.$len.' bytes from '. + $self->{host}.':'.$self->{port}); + + } elsif (($sz = length($buf)) < $len) { + + $pre .= $buf; + $len -= $sz; + + } else { + return $pre.$buf; + } + } +} + +# +# Read from the socket +# +# @param int $len How many bytes +# @return string Binary data +# +sub read +{ + my $self = shift; + my $len = shift; + + return unless defined $self->{handle}; + + #check for timeout + my @sockets = $self->{handle}->can_read( $self->{sendTimeout} / 1000 ); + + if(@sockets == 0){ + die new Thrift::TException('TSocket: timed out reading '.$len.' bytes from '. + $self->{host}.':'.$self->{port}); + } + + my $sock = $sockets[0]; + + my ($buf,$sz); + $sock->recv($buf, $len); + + if (!defined $buf || $buf eq '') { + + die new TException('TSocket: Could not read '.$len.' bytes from '. + $self->{host}.':'.$self->{port}); + + } + + return $buf; +} + + +# +# Write to the socket. +# +# @param string $buf The data to write +# +sub write +{ + my $self = shift; + my $buf = shift; + + + return unless defined $self->{handle}; + + while (length($buf) > 0) { + + + #check for timeout + my @sockets = $self->{handle}->can_write( $self->{recvTimeout} / 1000 ); + + if(@sockets == 0){ + die new Thrift::TException('TSocket: timed out writing to bytes from '. + $self->{host}.':'.$self->{port}); + } + + my $sock = $sockets[0]; + + my $got = $sock->send($buf); + + if (!defined $got || $got == 0 ) { + die new Thrift::TException('TSocket: Could not write '.length($buf).' bytes '. + $self->{host}.':'.$self->{host}); + } + + $buf = substr($buf, $got); + } +} + +# +# Flush output to the socket. +# +sub flush +{ + my $self = shift; + + return unless defined $self->{handle}; + + my $ret = ($self->{handle}->handles())[0]->flush; +} + +1; diff --git a/lib/perl/lib/Thrift/Transport.pm b/lib/perl/lib/Thrift/Transport.pm new file mode 100644 index 00000000..e22592be --- /dev/null +++ b/lib/perl/lib/Thrift/Transport.pm @@ -0,0 +1,129 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 5.6.0; +use strict; +use warnings; + +use Thrift; + +# +# Transport exceptions +# +package TTransportException; +use base('Thrift::TException'); + +use constant UNKNOWN => 0; +use constant NOT_OPEN => 1; +use constant ALREADY_OPEN => 2; +use constant TIMED_OUT => 3; +use constant END_OF_FILE => 4; + +sub new{ + my $classname = shift; + my $self = $classname->SUPER::new(@_); + + return bless($self,$classname); +} + +package Thrift::Transport; + +# +# Whether this transport is open. +# +# @return boolean true if open +# +sub isOpen +{ + die "abstract"; +} + +# +# Open the transport for reading/writing +# +# @throws TTransportException if cannot open +# +sub open +{ + die "abstract"; +} + +# +# Close the transport. +# +sub close +{ + die "abstract"; +} + +# +# Read some data into the array. +# +# @param int $len How much to read +# @return string The data that has been read +# @throws TTransportException if cannot read any more data +# +sub read +{ + my ($len); + die("abstract"); +} + +# +# Guarantees that the full amount of data is read. +# +# @return string The data, of exact length +# @throws TTransportException if cannot read data +# +sub readAll +{ + my $self = shift; + my $len = shift; + + my $data = ''; + my $got = 0; + + while (($got = length($data)) < $len) { + $data .= $self->read($len - $got); + } + + return $data; +} + +# +# Writes the given data out. +# +# @param string $buf The data to write +# @throws TTransportException if writing fails +# +sub write +{ + my ($buf); + die "abstract"; +} + +# +# Flushes any pending data out of a buffer +# +# @throws TTransportException if a writing error occurs +# +sub flush {} + +1; + diff --git a/lib/perl/test.pl b/lib/perl/test.pl new file mode 100644 index 00000000..7e068402 --- /dev/null +++ b/lib/perl/test.pl @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +use strict; +use warnings; + +use Test::Harness; + +runtests(@ARGV); diff --git a/lib/perl/test/Makefile.am b/lib/perl/test/Makefile.am new file mode 100644 index 00000000..ce87c48d --- /dev/null +++ b/lib/perl/test/Makefile.am @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +THRIFT = @top_builddir@/compiler/cpp/thrift +THRIFT_IF = @top_srcdir@/test/ThriftTest.thrift + +check-local: gen-perl/ThriftTest/Types.pm + +gen-perl/ThriftTest/Types.pm: $(THRIFT_IF) + $(THRIFT) --gen perl $(THRIFT_IF) + +clean-local: + rm -rf gen-perl + +EXTRA_DIST = memory_buffer.t diff --git a/lib/perl/test/memory_buffer.t b/lib/perl/test/memory_buffer.t new file mode 100644 index 00000000..8fa9fd72 --- /dev/null +++ b/lib/perl/test/memory_buffer.t @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +use Test::More tests => 6; + +use strict; +use warnings; + +use Data::Dumper; + +use Thrift::BinaryProtocol; +use Thrift::MemoryBuffer; + +use ThriftTest::Types; + + +my $transport = Thrift::MemoryBuffer->new(); +my $protocol = Thrift::BinaryProtocol->new($transport); + +my $a = ThriftTest::Xtruct->new(); +$a->i32_thing(10); +$a->i64_thing(30); +$a->string_thing('Hello, world!'); +$a->write($protocol); + +my $b = ThriftTest::Xtruct->new(); +$b->read($protocol); +is($b->i32_thing, $a->i32_thing); +is($b->i64_thing, $a->i64_thing); +is($b->string_thing, $a->string_thing); + +$b->write($protocol); +my $c = ThriftTest::Xtruct->new(); +$c->read($protocol); +is($c->i32_thing, $a->i32_thing); +is($c->i64_thing, $a->i64_thing); +is($c->string_thing, $a->string_thing); diff --git a/lib/php/README b/lib/php/README new file mode 100644 index 00000000..bb566f42 --- /dev/null +++ b/lib/php/README @@ -0,0 +1,63 @@ +Thrift PHP Software Library + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Using Thrift with PHP +===================== + +Thrift requires PHP 5. Thrift makes as few assumptions about your PHP +environment as possible while trying to make some more advanced PHP +features (i.e. APC cacheing using asbolute path URLs) as simple as possible. + +To use Thrift in your PHP codebase, take the following steps: + +#1) Copy all of thrift/lib/php/src into your PHP codebase +#2) Set $GLOBALS['THRIFT_ROOT'] to the path you installed Thrift +#3) include_once $GLOBALS['THRIFT_ROOT'].'/Thrift.php'; + +Note that #3 must be done before including any other Thrift files. +If you do not do #2, Thrift.php will set this global for you, but it will be +done using dirname(__FILE__), which is less efficient than providing the static +string yourself. + +When you generate a Thrift package using the compiler, it makes an assumption +about where your generated code will live. If your file is "MyPackage.thrift", +the generated files must be installed into: + +$GLOBALS['THRIFT_ROOT'].'/packages/MyPackage/'; + +This allows the code generator to compile your code without any extra flags +for the target directory names while still allowing your include paths to +be absolute (if you have an absolute THRIFT_ROOT). + +Dependencies +============ + +PHP_INT_SIZE + + This built-in signals whether your architecture is 32 or 64 bit and is + used by the TBinaryProtocol to properly use pack() and unpack() to + serialize data. + +apc_fetch(), apc_store() + + APC cache is used by the TSocketPool class. If you do not have APC installed, + Thrift will fill in null stub function definitions. diff --git a/lib/php/README.apache b/lib/php/README.apache new file mode 100644 index 00000000..8c41833d --- /dev/null +++ b/lib/php/README.apache @@ -0,0 +1,62 @@ +Thrift PHP/Apache Integration + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Building PHP Thrift Services with Apache +======================================== + +Thrift can be embedded in the Apache webserver with PHP installed. Sample +code is provided below. Note that to make requests to this type of server +you must use a THttpClient transport. + +Sample Code +=========== + +open(); +$processor->process($protocol, $protocol); +$transport->close(); diff --git a/lib/php/src/Thrift.php b/lib/php/src/Thrift.php new file mode 100644 index 00000000..ef6ab8a4 --- /dev/null +++ b/lib/php/src/Thrift.php @@ -0,0 +1,787 @@ + $fspec) { + $var = $fspec['var']; + if (isset($vals[$var])) { + $this->$var = $vals[$var]; + } + } + } else { + parent::__construct($p1, $p2); + } + } + + static $tmethod = array(TType::BOOL => 'Bool', + TType::BYTE => 'Byte', + TType::I16 => 'I16', + TType::I32 => 'I32', + TType::I64 => 'I64', + TType::DOUBLE => 'Double', + TType::STRING => 'String'); + + private function _readMap(&$var, $spec, $input) { + $xfer = 0; + $ktype = $spec['ktype']; + $vtype = $spec['vtype']; + $kread = $vread = null; + if (isset(TBase::$tmethod[$ktype])) { + $kread = 'read'.TBase::$tmethod[$ktype]; + } else { + $kspec = $spec['key']; + } + if (isset(TBase::$tmethod[$vtype])) { + $vread = 'read'.TBase::$tmethod[$vtype]; + } else { + $vspec = $spec['val']; + } + $var = array(); + $_ktype = $_vtype = $size = 0; + $xfer += $input->readMapBegin($_ktype, $_vtype, $size); + for ($i = 0; $i < $size; ++$i) { + $key = $val = null; + if ($kread !== null) { + $xfer += $input->$kread($key); + } else { + switch ($ktype) { + case TType::STRUCT: + $class = $kspec['class']; + $key = new $class(); + $xfer += $key->read($input); + break; + case TType::MAP: + $xfer += $this->_readMap($key, $kspec, $input); + break; + case TType::LST: + $xfer += $this->_readList($key, $kspec, $input, false); + break; + case TType::SET: + $xfer += $this->_readList($key, $kspec, $input, true); + break; + } + } + if ($vread !== null) { + $xfer += $input->$vread($val); + } else { + switch ($vtype) { + case TType::STRUCT: + $class = $vspec['class']; + $val = new $class(); + $xfer += $val->read($input); + break; + case TType::MAP: + $xfer += $this->_readMap($val, $vspec, $input); + break; + case TType::LST: + $xfer += $this->_readList($val, $vspec, $input, false); + break; + case TType::SET: + $xfer += $this->_readList($val, $vspec, $input, true); + break; + } + } + $var[$key] = $val; + } + $xfer += $input->readMapEnd(); + return $xfer; + } + + private function _readList(&$var, $spec, $input, $set=false) { + $xfer = 0; + $etype = $spec['etype']; + $eread = $vread = null; + if (isset(TBase::$tmethod[$etype])) { + $eread = 'read'.TBase::$tmethod[$etype]; + } else { + $espec = $spec['elem']; + } + $var = array(); + $_etype = $size = 0; + if ($set) { + $xfer += $input->readSetBegin($_etype, $size); + } else { + $xfer += $input->readListBegin($_etype, $size); + } + for ($i = 0; $i < $size; ++$i) { + $elem = null; + if ($eread !== null) { + $xfer += $input->$eread($elem); + } else { + $espec = $spec['elem']; + switch ($etype) { + case TType::STRUCT: + $class = $espec['class']; + $elem = new $class(); + $xfer += $elem->read($input); + break; + case TType::MAP: + $xfer += $this->_readMap($elem, $espec, $input); + break; + case TType::LST: + $xfer += $this->_readList($elem, $espec, $input, false); + break; + case TType::SET: + $xfer += $this->_readList($elem, $espec, $input, true); + break; + } + } + if ($set) { + $var[$elem] = true; + } else { + $var []= $elem; + } + } + if ($set) { + $xfer += $input->readSetEnd(); + } else { + $xfer += $input->readListEnd(); + } + return $xfer; + } + + protected function _read($class, $spec, $input) { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + if (isset($spec[$fid])) { + $fspec = $spec[$fid]; + $var = $fspec['var']; + if ($ftype == $fspec['type']) { + $xfer = 0; + if (isset(TBase::$tmethod[$ftype])) { + $func = 'read'.TBase::$tmethod[$ftype]; + $xfer += $input->$func($this->$var); + } else { + switch ($ftype) { + case TType::STRUCT: + $class = $fspec['class']; + $this->$var = new $class(); + $xfer += $this->$var->read($input); + break; + case TType::MAP: + $xfer += $this->_readMap($this->$var, $fspec, $input); + break; + case TType::LST: + $xfer += $this->_readList($this->$var, $fspec, $input, false); + break; + case TType::SET: + $xfer += $this->_readList($this->$var, $fspec, $input, true); + break; + } + } + } else { + $xfer += $input->skip($ftype); + } + } else { + $xfer += $input->skip($ftype); + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + private function _writeMap($var, $spec, $output) { + $xfer = 0; + $ktype = $spec['ktype']; + $vtype = $spec['vtype']; + $kwrite = $vwrite = null; + if (isset(TBase::$tmethod[$ktype])) { + $kwrite = 'write'.TBase::$tmethod[$ktype]; + } else { + $kspec = $spec['key']; + } + if (isset(TBase::$tmethod[$vtype])) { + $vwrite = 'write'.TBase::$tmethod[$vtype]; + } else { + $vspec = $spec['val']; + } + $xfer += $output->writeMapBegin($ktype, $vtype, count($var)); + foreach ($var as $key => $val) { + if (isset($kwrite)) { + $xfer += $output->$kwrite($key); + } else { + switch ($ktype) { + case TType::STRUCT: + $xfer += $key->write($output); + break; + case TType::MAP: + $xfer += $this->_writeMap($key, $kspec, $output); + break; + case TType::LST: + $xfer += $this->_writeList($key, $kspec, $output, false); + break; + case TType::SET: + $xfer += $this->_writeList($key, $kspec, $output, true); + break; + } + } + if (isset($vwrite)) { + $xfer += $output->$vwrite($val); + } else { + switch ($vtype) { + case TType::STRUCT: + $xfer += $val->write($output); + break; + case TType::MAP: + $xfer += $this->_writeMap($val, $vspec, $output); + break; + case TType::LST: + $xfer += $this->_writeList($val, $vspec, $output, false); + break; + case TType::SET: + $xfer += $this->_writeList($val, $vspec, $output, true); + break; + } + } + } + $xfer += $output->writeMapEnd(); + return $xfer; + } + + private function _writeList($var, $spec, $output, $set=false) { + $xfer = 0; + $etype = $spec['etype']; + $ewrite = null; + if (isset(TBase::$tmethod[$etype])) { + $ewrite = 'write'.TBase::$tmethod[$etype]; + } else { + $espec = $spec['elem']; + } + if ($set) { + $xfer += $output->writeSetBegin($etype, count($var)); + } else { + $xfer += $output->writeListBegin($etype, count($var)); + } + foreach ($var as $key => $val) { + $elem = $set ? $key : $val; + if (isset($ewrite)) { + $xfer += $output->$ewrite($elem); + } else { + switch ($etype) { + case TType::STRUCT: + $xfer += $elem->write($output); + break; + case TType::MAP: + $xfer += $this->_writeMap($elem, $espec, $output); + break; + case TType::LST: + $xfer += $this->_writeList($elem, $espec, $output, false); + break; + case TType::SET: + $xfer += $this->_writeList($elem, $espec, $output, true); + break; + } + } + } + if ($set) { + $xfer += $output->writeSetEnd(); + } else { + $xfer += $output->writeListEnd(); + } + return $xfer; + } + + protected function _write($class, $spec, $output) { + $xfer = 0; + $xfer += $output->writeStructBegin($class); + foreach ($spec as $fid => $fspec) { + $var = $fspec['var']; + if ($this->$var !== null) { + $ftype = $fspec['type']; + $xfer += $output->writeFieldBegin($var, $ftype, $fid); + if (isset(TBase::$tmethod[$ftype])) { + $func = 'write'.TBase::$tmethod[$ftype]; + $xfer += $output->$func($this->$var); + } else { + switch ($ftype) { + case TType::STRUCT: + $xfer += $this->$var->write($output); + break; + case TType::MAP: + $xfer += $this->_writeMap($this->$var, $fspec, $output); + break; + case TType::LST: + $xfer += $this->_writeList($this->$var, $fspec, $output, false); + break; + case TType::SET: + $xfer += $this->_writeList($this->$var, $fspec, $output, true); + break; + } + } + $xfer += $output->writeFieldEnd(); + } + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } + +} + +/** + * Base class from which other Thrift structs extend. This is so that we can + * cut back on the size of the generated code which is turning out to have a + * nontrivial cost just to load thanks to the wondrously abysmal implementation + * of PHP. Note that code is intentionally duplicated in here to avoid making + * function calls for every field or member of a container.. + */ +abstract class TBase { + + static $tmethod = array(TType::BOOL => 'Bool', + TType::BYTE => 'Byte', + TType::I16 => 'I16', + TType::I32 => 'I32', + TType::I64 => 'I64', + TType::DOUBLE => 'Double', + TType::STRING => 'String'); + + abstract function read($input); + + abstract function write($output); + + public function __construct($spec=null, $vals=null) { + if (is_array($spec) && is_array($vals)) { + foreach ($spec as $fid => $fspec) { + $var = $fspec['var']; + if (isset($vals[$var])) { + $this->$var = $vals[$var]; + } + } + } + } + + private function _readMap(&$var, $spec, $input) { + $xfer = 0; + $ktype = $spec['ktype']; + $vtype = $spec['vtype']; + $kread = $vread = null; + if (isset(TBase::$tmethod[$ktype])) { + $kread = 'read'.TBase::$tmethod[$ktype]; + } else { + $kspec = $spec['key']; + } + if (isset(TBase::$tmethod[$vtype])) { + $vread = 'read'.TBase::$tmethod[$vtype]; + } else { + $vspec = $spec['val']; + } + $var = array(); + $_ktype = $_vtype = $size = 0; + $xfer += $input->readMapBegin($_ktype, $_vtype, $size); + for ($i = 0; $i < $size; ++$i) { + $key = $val = null; + if ($kread !== null) { + $xfer += $input->$kread($key); + } else { + switch ($ktype) { + case TType::STRUCT: + $class = $kspec['class']; + $key = new $class(); + $xfer += $key->read($input); + break; + case TType::MAP: + $xfer += $this->_readMap($key, $kspec, $input); + break; + case TType::LST: + $xfer += $this->_readList($key, $kspec, $input, false); + break; + case TType::SET: + $xfer += $this->_readList($key, $kspec, $input, true); + break; + } + } + if ($vread !== null) { + $xfer += $input->$vread($val); + } else { + switch ($vtype) { + case TType::STRUCT: + $class = $vspec['class']; + $val = new $class(); + $xfer += $val->read($input); + break; + case TType::MAP: + $xfer += $this->_readMap($val, $vspec, $input); + break; + case TType::LST: + $xfer += $this->_readList($val, $vspec, $input, false); + break; + case TType::SET: + $xfer += $this->_readList($val, $vspec, $input, true); + break; + } + } + $var[$key] = $val; + } + $xfer += $input->readMapEnd(); + return $xfer; + } + + private function _readList(&$var, $spec, $input, $set=false) { + $xfer = 0; + $etype = $spec['etype']; + $eread = $vread = null; + if (isset(TBase::$tmethod[$etype])) { + $eread = 'read'.TBase::$tmethod[$etype]; + } else { + $espec = $spec['elem']; + } + $var = array(); + $_etype = $size = 0; + if ($set) { + $xfer += $input->readSetBegin($_etype, $size); + } else { + $xfer += $input->readListBegin($_etype, $size); + } + for ($i = 0; $i < $size; ++$i) { + $elem = null; + if ($eread !== null) { + $xfer += $input->$eread($elem); + } else { + $espec = $spec['elem']; + switch ($etype) { + case TType::STRUCT: + $class = $espec['class']; + $elem = new $class(); + $xfer += $elem->read($input); + break; + case TType::MAP: + $xfer += $this->_readMap($elem, $espec, $input); + break; + case TType::LST: + $xfer += $this->_readList($elem, $espec, $input, false); + break; + case TType::SET: + $xfer += $this->_readList($elem, $espec, $input, true); + break; + } + } + if ($set) { + $var[$elem] = true; + } else { + $var []= $elem; + } + } + if ($set) { + $xfer += $input->readSetEnd(); + } else { + $xfer += $input->readListEnd(); + } + return $xfer; + } + + protected function _read($class, $spec, $input) { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + if (isset($spec[$fid])) { + $fspec = $spec[$fid]; + $var = $fspec['var']; + if ($ftype == $fspec['type']) { + $xfer = 0; + if (isset(TBase::$tmethod[$ftype])) { + $func = 'read'.TBase::$tmethod[$ftype]; + $xfer += $input->$func($this->$var); + } else { + switch ($ftype) { + case TType::STRUCT: + $class = $fspec['class']; + $this->$var = new $class(); + $xfer += $this->$var->read($input); + break; + case TType::MAP: + $xfer += $this->_readMap($this->$var, $fspec, $input); + break; + case TType::LST: + $xfer += $this->_readList($this->$var, $fspec, $input, false); + break; + case TType::SET: + $xfer += $this->_readList($this->$var, $fspec, $input, true); + break; + } + } + } else { + $xfer += $input->skip($ftype); + } + } else { + $xfer += $input->skip($ftype); + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + private function _writeMap($var, $spec, $output) { + $xfer = 0; + $ktype = $spec['ktype']; + $vtype = $spec['vtype']; + $kwrite = $vwrite = null; + if (isset(TBase::$tmethod[$ktype])) { + $kwrite = 'write'.TBase::$tmethod[$ktype]; + } else { + $kspec = $spec['key']; + } + if (isset(TBase::$tmethod[$vtype])) { + $vwrite = 'write'.TBase::$tmethod[$vtype]; + } else { + $vspec = $spec['val']; + } + $xfer += $output->writeMapBegin($ktype, $vtype, count($var)); + foreach ($var as $key => $val) { + if (isset($kwrite)) { + $xfer += $output->$kwrite($key); + } else { + switch ($ktype) { + case TType::STRUCT: + $xfer += $key->write($output); + break; + case TType::MAP: + $xfer += $this->_writeMap($key, $kspec, $output); + break; + case TType::LST: + $xfer += $this->_writeList($key, $kspec, $output, false); + break; + case TType::SET: + $xfer += $this->_writeList($key, $kspec, $output, true); + break; + } + } + if (isset($vwrite)) { + $xfer += $output->$vwrite($val); + } else { + switch ($vtype) { + case TType::STRUCT: + $xfer += $val->write($output); + break; + case TType::MAP: + $xfer += $this->_writeMap($val, $vspec, $output); + break; + case TType::LST: + $xfer += $this->_writeList($val, $vspec, $output, false); + break; + case TType::SET: + $xfer += $this->_writeList($val, $vspec, $output, true); + break; + } + } + } + $xfer += $output->writeMapEnd(); + return $xfer; + } + + private function _writeList($var, $spec, $output, $set=false) { + $xfer = 0; + $etype = $spec['etype']; + $ewrite = null; + if (isset(TBase::$tmethod[$etype])) { + $ewrite = 'write'.TBase::$tmethod[$etype]; + } else { + $espec = $spec['elem']; + } + if ($set) { + $xfer += $output->writeSetBegin($etype, count($var)); + } else { + $xfer += $output->writeListBegin($etype, count($var)); + } + foreach ($var as $key => $val) { + $elem = $set ? $key : $val; + if (isset($ewrite)) { + $xfer += $output->$ewrite($elem); + } else { + switch ($etype) { + case TType::STRUCT: + $xfer += $elem->write($output); + break; + case TType::MAP: + $xfer += $this->_writeMap($elem, $espec, $output); + break; + case TType::LST: + $xfer += $this->_writeList($elem, $espec, $output, false); + break; + case TType::SET: + $xfer += $this->_writeList($elem, $espec, $output, true); + break; + } + } + } + if ($set) { + $xfer += $output->writeSetEnd(); + } else { + $xfer += $output->writeListEnd(); + } + return $xfer; + } + + protected function _write($class, $spec, $output) { + $xfer = 0; + $xfer += $output->writeStructBegin($class); + foreach ($spec as $fid => $fspec) { + $var = $fspec['var']; + if ($this->$var !== null) { + $ftype = $fspec['type']; + $xfer += $output->writeFieldBegin($var, $ftype, $fid); + if (isset(TBase::$tmethod[$ftype])) { + $func = 'write'.TBase::$tmethod[$ftype]; + $xfer += $output->$func($this->$var); + } else { + switch ($ftype) { + case TType::STRUCT: + $xfer += $this->$var->write($output); + break; + case TType::MAP: + $xfer += $this->_writeMap($this->$var, $fspec, $output); + break; + case TType::LST: + $xfer += $this->_writeList($this->$var, $fspec, $output, false); + break; + case TType::SET: + $xfer += $this->_writeList($this->$var, $fspec, $output, true); + break; + } + } + $xfer += $output->writeFieldEnd(); + } + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } +} + +class TApplicationException extends TException { + static $_TSPEC = + array(1 => array('var' => 'message', + 'type' => TType::STRING), + 2 => array('var' => 'code', + 'type' => TType::I32)); + + const UNKNOWN = 0; + const UNKNOWN_METHOD = 1; + const INVALID_MESSAGE_TYPE = 2; + const WRONG_METHOD_NAME = 3; + const BAD_SEQUENCE_ID = 4; + const MISSING_RESULT = 5; + + function __construct($message=null, $code=0) { + parent::__construct($message, $code); + } + + public function read($output) { + return $this->_read('TApplicationException', self::$_TSPEC, $output); + } + + public function write($output) { + $xfer = 0; + $xfer += $output->writeStructBegin('TApplicationException'); + if ($message = $this->getMessage()) { + $xfer += $output->writeFieldBegin('message', TType::STRING, 1); + $xfer += $output->writeString($message); + $xfer += $output->writeFieldEnd(); + } + if ($code = $this->getCode()) { + $xfer += $output->writeFieldBegin('type', TType::I32, 2); + $xfer += $output->writeI32($code); + $xfer += $output->writeFieldEnd(); + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } +} + +/** + * Set global THRIFT ROOT automatically via inclusion here + */ +if (!isset($GLOBALS['THRIFT_ROOT'])) { + $GLOBALS['THRIFT_ROOT'] = dirname(__FILE__); +} +include_once $GLOBALS['THRIFT_ROOT'].'/protocol/TProtocol.php'; +include_once $GLOBALS['THRIFT_ROOT'].'/transport/TTransport.php'; + +?> diff --git a/lib/php/src/autoload.php b/lib/php/src/autoload.php new file mode 100644 index 00000000..3a35545d --- /dev/null +++ b/lib/php/src/autoload.php @@ -0,0 +1,51 @@ + +#include +#include +#include +#include +#include + +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define htonll(x) bswap_64(x) +#define ntohll(x) bswap_64(x) +#else +#define htonll(x) x +#define ntohll(x) x +#endif + +enum TType { + T_STOP = 0, + T_VOID = 1, + T_BOOL = 2, + T_BYTE = 3, + T_I08 = 3, + T_I16 = 6, + T_I32 = 8, + T_U64 = 9, + T_I64 = 10, + T_DOUBLE = 4, + T_STRING = 11, + T_UTF7 = 11, + T_STRUCT = 12, + T_MAP = 13, + T_SET = 14, + T_LIST = 15, + T_UTF8 = 16, + T_UTF16 = 17 +}; + +const int32_t VERSION_MASK = 0xffff0000; +const int32_t VERSION_1 = 0x80010000; +const int8_t T_CALL = 1; +const int8_t T_REPLY = 2; +const int8_t T_EXCEPTION = 3; +// tprotocolexception +const int INVALID_DATA = 1; +const int BAD_VERSION = 4; + +#include "php.h" +#include "zend_interfaces.h" +#include "zend_exceptions.h" +#include "php_thrift_protocol.h" + +static function_entry thrift_protocol_functions[] = { + PHP_FE(thrift_protocol_write_binary, NULL) + PHP_FE(thrift_protocol_read_binary, NULL) + {NULL, NULL, NULL} +} ; + +zend_module_entry thrift_protocol_module_entry = { + STANDARD_MODULE_HEADER, + "thrift_protocol", + thrift_protocol_functions, + NULL, + NULL, + NULL, + NULL, + NULL, + "1.0", + STANDARD_MODULE_PROPERTIES +}; + +#ifdef COMPILE_DL_THRIFT_PROTOCOL +ZEND_GET_MODULE(thrift_protocol) +#endif + +class PHPExceptionWrapper : public std::exception { +public: + PHPExceptionWrapper(zval* _ex) throw() : ex(_ex) { + snprintf(_what, 40, "PHP exception zval=%p", ex); + } + const char* what() const throw() { return _what; } + ~PHPExceptionWrapper() throw() {} + operator zval*() const throw() { return const_cast(ex); } // Zend API doesn't do 'const'... +protected: + zval* ex; + char _what[40]; +} ; + +class PHPTransport { +public: + zval* protocol() { return p; } + zval* transport() { return t; } +protected: + PHPTransport() {} + + void construct_with_zval(zval* _p, size_t _buffer_size) { + buffer = reinterpret_cast(emalloc(_buffer_size)); + buffer_ptr = buffer; + buffer_used = 0; + buffer_size = _buffer_size; + p = _p; + + // Get the transport for the passed protocol + zval gettransport; + ZVAL_STRING(&gettransport, "getTransport", 0); + MAKE_STD_ZVAL(t); + ZVAL_NULL(t); + TSRMLS_FETCH(); + call_user_function(EG(function_table), &p, &gettransport, t, 0, NULL TSRMLS_CC); + } + ~PHPTransport() { + efree(buffer); + zval_ptr_dtor(&t); + } + + char* buffer; + char* buffer_ptr; + size_t buffer_used; + size_t buffer_size; + + zval* p; + zval* t; +}; + + +class PHPOutputTransport : public PHPTransport { +public: + PHPOutputTransport(zval* _p, size_t _buffer_size = 8192) { + construct_with_zval(_p, _buffer_size); + } + + ~PHPOutputTransport() { + flush(); + directFlush(); + } + + void write(const char* data, size_t len) { + if ((len + buffer_used) > buffer_size) { + flush(); + } + if (len > buffer_size) { + directWrite(data, len); + } else { + memcpy(buffer_ptr, data, len); + buffer_used += len; + buffer_ptr += len; + } + } + + void writeI64(int64_t i) { + i = htonll(i); + write((const char*)&i, 8); + } + + void writeU32(uint32_t i) { + i = htonl(i); + write((const char*)&i, 4); + } + + void writeI32(int32_t i) { + i = htonl(i); + write((const char*)&i, 4); + } + + void writeI16(int16_t i) { + i = htons(i); + write((const char*)&i, 2); + } + + void writeI8(int8_t i) { + write((const char*)&i, 1); + } + + void writeString(const char* str, size_t len) { + writeU32(len); + write(str, len); + } + + void flush() { + if (buffer_used) { + directWrite(buffer, buffer_used); + buffer_ptr = buffer; + buffer_used = 0; + } + } + +protected: + void directFlush() { + zval ret; + ZVAL_NULL(&ret); + zval flushfn; + ZVAL_STRING(&flushfn, "flush", 0); + TSRMLS_FETCH(); + call_user_function(EG(function_table), &t, &flushfn, &ret, 0, NULL TSRMLS_CC); + zval_dtor(&ret); + } + void directWrite(const char* data, size_t len) { + zval writefn; + ZVAL_STRING(&writefn, "write", 0); + char* newbuf = (char*)emalloc(buffer_used + 1); + memcpy(newbuf, buffer, buffer_used); + newbuf[buffer_used] = '\0'; + zval *args[1]; + MAKE_STD_ZVAL(args[0]); + ZVAL_STRINGL(args[0], newbuf, buffer_used, 0); + TSRMLS_FETCH(); + zval ret; + ZVAL_NULL(&ret); + call_user_function(EG(function_table), &t, &writefn, &ret, 1, args TSRMLS_CC); + zval_ptr_dtor(args); + zval_dtor(&ret); + if (EG(exception)) { + zval* ex = EG(exception); + EG(exception) = NULL; + throw PHPExceptionWrapper(ex); + } + } +}; + +class PHPInputTransport : public PHPTransport { +public: + PHPInputTransport(zval* _p, size_t _buffer_size = 8192) { + construct_with_zval(_p, _buffer_size); + } + + ~PHPInputTransport() { + put_back(); + } + + void put_back() { + if (buffer_used) { + zval putbackfn; + ZVAL_STRING(&putbackfn, "putBack", 0); + + char* newbuf = (char*)emalloc(buffer_used + 1); + memcpy(newbuf, buffer_ptr, buffer_used); + newbuf[buffer_used] = '\0'; + + zval *args[1]; + MAKE_STD_ZVAL(args[0]); + ZVAL_STRINGL(args[0], newbuf, buffer_used, 0); + + TSRMLS_FETCH(); + + zval ret; + ZVAL_NULL(&ret); + call_user_function(EG(function_table), &t, &putbackfn, &ret, 1, args TSRMLS_CC); + zval_ptr_dtor(args); + zval_dtor(&ret); + } + buffer_used = 0; + buffer_ptr = buffer; + } + + void skip(size_t len) { + while (len) { + size_t chunk_size = MIN(len, buffer_used); + if (chunk_size) { + buffer_ptr = reinterpret_cast(buffer_ptr) + chunk_size; + buffer_used -= chunk_size; + len -= chunk_size; + } + if (! len) break; + refill(); + } + } + + void readBytes(void* buf, size_t len) { + while (len) { + size_t chunk_size = MIN(len, buffer_used); + if (chunk_size) { + memcpy(buf, buffer_ptr, chunk_size); + buffer_ptr = reinterpret_cast(buffer_ptr) + chunk_size; + buffer_used -= chunk_size; + buf = reinterpret_cast(buf) + chunk_size; + len -= chunk_size; + } + if (! len) break; + refill(); + } + } + + int8_t readI8() { + int8_t c; + readBytes(&c, 1); + return c; + } + + int16_t readI16() { + int16_t c; + readBytes(&c, 2); + return (int16_t)ntohs(c); + } + + uint32_t readU32() { + uint32_t c; + readBytes(&c, 4); + return (uint32_t)ntohl(c); + } + + int32_t readI32() { + int32_t c; + readBytes(&c, 4); + return (int32_t)ntohl(c); + } + +protected: + void refill() { + assert(buffer_used == 0); + zval retval; + ZVAL_NULL(&retval); + + zval *args[1]; + MAKE_STD_ZVAL(args[0]); + ZVAL_LONG(args[0], buffer_size); + + TSRMLS_FETCH(); + + zval funcname; + ZVAL_STRING(&funcname, "read", 0); + + call_user_function(EG(function_table), &t, &funcname, &retval, 1, args TSRMLS_CC); + zval_ptr_dtor(args); + + if (EG(exception)) { + zval_dtor(&retval); + zval* ex = EG(exception); + EG(exception) = NULL; + throw PHPExceptionWrapper(ex); + } + + buffer_used = Z_STRLEN(retval); + memcpy(buffer, Z_STRVAL(retval), buffer_used); + zval_dtor(&retval); + + buffer_ptr = buffer; + } + +}; + +void binary_deserialize_spec(zval* zthis, PHPInputTransport& transport, HashTable* spec); +void binary_serialize_spec(zval* zthis, PHPOutputTransport& transport, HashTable* spec); +void binary_serialize(int8_t thrift_typeID, PHPOutputTransport& transport, zval** value, HashTable* fieldspec); +void skip_element(long thrift_typeID, PHPInputTransport& transport); + +// Create a PHP object given a typename and call the ctor, optionally passing up to 2 arguments +void createObject(char* obj_typename, zval* return_value, int nargs = 0, zval* arg1 = NULL, zval* arg2 = NULL) { + TSRMLS_FETCH(); + size_t obj_typename_len = strlen(obj_typename); + zend_class_entry* ce = zend_fetch_class(obj_typename, obj_typename_len, ZEND_FETCH_CLASS_DEFAULT TSRMLS_CC); + if (! ce) { + php_error_docref(NULL TSRMLS_CC, E_ERROR, "Class %s does not exist", obj_typename); + RETURN_NULL(); + } + + object_and_properties_init(return_value, ce, NULL); + zend_function* constructor = zend_std_get_constructor(return_value TSRMLS_CC); + zval* ctor_rv = NULL; + zend_call_method(&return_value, ce, &constructor, NULL, 0, &ctor_rv, nargs, arg1, arg2 TSRMLS_CC); + zval_ptr_dtor(&ctor_rv); +} + +void throw_tprotocolexception(char* what, long errorcode) { + TSRMLS_FETCH(); + + zval *zwhat, *zerrorcode; + MAKE_STD_ZVAL(zwhat); + MAKE_STD_ZVAL(zerrorcode); + + ZVAL_STRING(zwhat, what, 1); + ZVAL_LONG(zerrorcode, errorcode); + + zval* ex; + MAKE_STD_ZVAL(ex); + createObject("TProtocolException", ex, 2, zwhat, zerrorcode); + zval_ptr_dtor(&zwhat); + zval_ptr_dtor(&zerrorcode); + throw PHPExceptionWrapper(ex); +} + +void binary_deserialize(int8_t thrift_typeID, PHPInputTransport& transport, zval* return_value, HashTable* fieldspec) { + zval** val_ptr; + Z_TYPE_P(return_value) = IS_NULL; // just in case + + switch (thrift_typeID) { + case T_STOP: + case T_VOID: + RETURN_NULL(); + return; + case T_STRUCT: { + if (zend_hash_find(fieldspec, "class", 6, (void**)&val_ptr) != SUCCESS) { + throw_tprotocolexception("no class type in spec", INVALID_DATA); + skip_element(T_STRUCT, transport); + RETURN_NULL(); + } + char* structType = Z_STRVAL_PP(val_ptr); + createObject(structType, return_value); + if (Z_TYPE_P(return_value) == IS_NULL) { + // unable to create class entry + skip_element(T_STRUCT, transport); + RETURN_NULL(); + } + TSRMLS_FETCH(); + zval* spec = zend_read_static_property(zend_get_class_entry(return_value TSRMLS_CC), "_TSPEC", 6, false TSRMLS_CC); + if (Z_TYPE_P(spec) != IS_ARRAY) { + char errbuf[128]; + snprintf(errbuf, 128, "spec for %s is wrong type: %d\n", structType, Z_TYPE_P(spec)); + throw_tprotocolexception(errbuf, INVALID_DATA); + RETURN_NULL(); + } + binary_deserialize_spec(return_value, transport, Z_ARRVAL_P(spec)); + return; + } break; + case T_BOOL: { + uint8_t c; + transport.readBytes(&c, 1); + RETURN_BOOL(c != 0); + } + //case T_I08: // same numeric value as T_BYTE + case T_BYTE: { + uint8_t c; + transport.readBytes(&c, 1); + RETURN_LONG(c); + } + case T_I16: { + uint16_t c; + transport.readBytes(&c, 2); + RETURN_LONG(ntohs(c)); + } + case T_I32: { + uint32_t c; + transport.readBytes(&c, 4); + RETURN_LONG(ntohl(c)); + } + case T_U64: + case T_I64: { + uint64_t c; + transport.readBytes(&c, 8); + RETURN_LONG(ntohll(c)); + } + case T_DOUBLE: { + union { + uint64_t c; + double d; + } a; + transport.readBytes(&(a.c), 8); + a.c = ntohll(a.c); + RETURN_DOUBLE(a.d); + } + //case T_UTF7: // aliases T_STRING + case T_UTF8: + case T_UTF16: + case T_STRING: { + uint32_t size = transport.readU32(); + if (size) { + char* strbuf = (char*) emalloc(size + 1); + transport.readBytes(strbuf, size); + strbuf[size] = '\0'; + ZVAL_STRINGL(return_value, strbuf, size, 0); + } else { + ZVAL_EMPTY_STRING(return_value); + } + return; + } + case T_MAP: { // array of key -> value + uint8_t types[2]; + transport.readBytes(types, 2); + uint32_t size = transport.readU32(); + array_init(return_value); + + zend_hash_find(fieldspec, "key", 4, (void**)&val_ptr); + HashTable* keyspec = Z_ARRVAL_PP(val_ptr); + zend_hash_find(fieldspec, "val", 4, (void**)&val_ptr); + HashTable* valspec = Z_ARRVAL_PP(val_ptr); + + for (uint32_t s = 0; s < size; ++s) { + zval *value; + MAKE_STD_ZVAL(value); + + zval* key; + MAKE_STD_ZVAL(key); + + binary_deserialize(types[0], transport, key, keyspec); + binary_deserialize(types[1], transport, value, valspec); + if (Z_TYPE_P(key) == IS_LONG) { + zend_hash_index_update(return_value->value.ht, Z_LVAL_P(key), &value, sizeof(zval *), NULL); + } + else { + if (Z_TYPE_P(key) != IS_STRING) convert_to_string(key); + zend_hash_update(return_value->value.ht, Z_STRVAL_P(key), Z_STRLEN_P(key) + 1, &value, sizeof(zval *), NULL); + } + zval_ptr_dtor(&key); + } + return; // return_value already populated + } + case T_LIST: { // array with autogenerated numeric keys + int8_t type = transport.readI8(); + uint32_t size = transport.readU32(); + zend_hash_find(fieldspec, "elem", 5, (void**)&val_ptr); + HashTable* elemspec = Z_ARRVAL_PP(val_ptr); + + array_init(return_value); + for (uint32_t s = 0; s < size; ++s) { + zval *value; + MAKE_STD_ZVAL(value); + binary_deserialize(type, transport, value, elemspec); + zend_hash_next_index_insert(return_value->value.ht, &value, sizeof(zval *), NULL); + } + return; + } + case T_SET: { // array of key -> TRUE + uint8_t type; + uint32_t size; + transport.readBytes(&type, 1); + transport.readBytes(&size, 4); + size = ntohl(size); + zend_hash_find(fieldspec, "elem", 5, (void**)&val_ptr); + HashTable* elemspec = Z_ARRVAL_PP(val_ptr); + + array_init(return_value); + + for (uint32_t s = 0; s < size; ++s) { + zval* key; + zval* value; + MAKE_STD_ZVAL(key); + MAKE_STD_ZVAL(value); + ZVAL_TRUE(value); + + binary_deserialize(type, transport, key, elemspec); + + if (Z_TYPE_P(key) == IS_LONG) { + zend_hash_index_update(return_value->value.ht, Z_LVAL_P(key), &value, sizeof(zval *), NULL); + } + else { + if (Z_TYPE_P(key) != IS_STRING) convert_to_string(key); + zend_hash_update(return_value->value.ht, Z_STRVAL_P(key), Z_STRLEN_P(key) + 1, &value, sizeof(zval *), NULL); + } + zval_ptr_dtor(&key); + } + return; + } + }; + + char errbuf[128]; + sprintf(errbuf, "Unknown thrift typeID %d", thrift_typeID); + throw_tprotocolexception(errbuf, INVALID_DATA); +} + +void skip_element(long thrift_typeID, PHPInputTransport& transport) { + switch (thrift_typeID) { + case T_STOP: + case T_VOID: + return; + case T_STRUCT: + while (true) { + int8_t ttype = transport.readI8(); // get field type + if (ttype == T_STOP) break; + transport.skip(2); // skip field number, I16 + skip_element(ttype, transport); // skip field payload + } + return; + case T_BOOL: + case T_BYTE: + transport.skip(1); + return; + case T_I16: + transport.skip(2); + return; + case T_I32: + transport.skip(4); + return; + case T_U64: + case T_I64: + case T_DOUBLE: + transport.skip(8); + return; + //case T_UTF7: // aliases T_STRING + case T_UTF8: + case T_UTF16: + case T_STRING: { + uint32_t len = transport.readU32(); + transport.skip(len); + } return; + case T_MAP: { + int8_t keytype = transport.readI8(); + int8_t valtype = transport.readI8(); + uint32_t size = transport.readU32(); + for (uint32_t i = 0; i < size; ++i) { + skip_element(keytype, transport); + skip_element(valtype, transport); + } + } return; + case T_LIST: + case T_SET: { + int8_t valtype = transport.readI8(); + uint32_t size = transport.readU32(); + for (uint32_t i = 0; i < size; ++i) { + skip_element(valtype, transport); + } + } return; + }; + + char errbuf[128]; + sprintf(errbuf, "Unknown thrift typeID %ld", thrift_typeID); + throw_tprotocolexception(errbuf, INVALID_DATA); +} + +void binary_serialize_hashtable_key(int8_t keytype, PHPOutputTransport& transport, HashTable* ht, HashPosition& ht_pos) { + bool keytype_is_numeric = (!((keytype == T_STRING) || (keytype == T_UTF8) || (keytype == T_UTF16))); + + char* key; + uint key_len; + long index = 0; + + zval* z; + MAKE_STD_ZVAL(z); + + int res = zend_hash_get_current_key_ex(ht, &key, &key_len, (ulong*)&index, 0, &ht_pos); + if (keytype_is_numeric) { + if (res == HASH_KEY_IS_STRING) { + index = strtol(key, NULL, 10); + } + ZVAL_LONG(z, index); + } else { + char buf[64]; + if (res == HASH_KEY_IS_STRING) { + key_len -= 1; // skip the null terminator + } else { + sprintf(buf, "%ld", index); + key = buf; key_len = strlen(buf); + } + ZVAL_STRINGL(z, key, key_len, 1); + } + binary_serialize(keytype, transport, &z, NULL); + zval_ptr_dtor(&z); +} + +inline bool ttype_is_int(int8_t t) { + return ((t == T_BYTE) || ((t >= T_I16) && (t <= T_I64))); +} + +inline bool ttypes_are_compatible(int8_t t1, int8_t t2) { + // Integer types of different widths are considered compatible; + // otherwise the typeID must match. + return ((t1 == t2) || (ttype_is_int(t1) && ttype_is_int(t2))); +} + +void binary_deserialize_spec(zval* zthis, PHPInputTransport& transport, HashTable* spec) { + // SET and LIST have 'elem' => array('type', [optional] 'class') + // MAP has 'val' => array('type', [optiona] 'class') + TSRMLS_FETCH(); + zend_class_entry* ce = zend_get_class_entry(zthis TSRMLS_CC); + while (true) { + zval** val_ptr = NULL; + + int8_t ttype = transport.readI8(); + if (ttype == T_STOP) return; + int16_t fieldno = transport.readI16(); + if (zend_hash_index_find(spec, fieldno, (void**)&val_ptr) == SUCCESS) { + HashTable* fieldspec = Z_ARRVAL_PP(val_ptr); + // pull the field name + // zend hash tables use the null at the end in the length... so strlen(hash key) + 1. + zend_hash_find(fieldspec, "var", 4, (void**)&val_ptr); + char* varname = Z_STRVAL_PP(val_ptr); + + // and the type + zend_hash_find(fieldspec, "type", 5, (void**)&val_ptr); + if (Z_TYPE_PP(val_ptr) != IS_LONG) convert_to_long(*val_ptr); + int8_t expected_ttype = Z_LVAL_PP(val_ptr); + + if (ttypes_are_compatible(ttype, expected_ttype)) { + zval* rv = NULL; + MAKE_STD_ZVAL(rv); + binary_deserialize(ttype, transport, rv, fieldspec); + zend_update_property(ce, zthis, varname, strlen(varname), rv TSRMLS_CC); + zval_ptr_dtor(&rv); + } else { + skip_element(ttype, transport); + } + } else { + skip_element(ttype, transport); + } + } +} + +void binary_serialize(int8_t thrift_typeID, PHPOutputTransport& transport, zval** value, HashTable* fieldspec) { + // At this point the typeID (and field num, if applicable) should've already been written to the output so all we need to do is write the payload. + switch (thrift_typeID) { + case T_STOP: + case T_VOID: + return; + case T_STRUCT: { + TSRMLS_FETCH(); + if (Z_TYPE_PP(value) != IS_OBJECT) { + throw_tprotocolexception("Attempt to send non-object type as a T_STRUCT", INVALID_DATA); + } + zval* spec = zend_read_static_property(zend_get_class_entry(*value TSRMLS_CC), "_TSPEC", 6, false TSRMLS_CC); + binary_serialize_spec(*value, transport, Z_ARRVAL_P(spec)); + } return; + case T_BOOL: + if (Z_TYPE_PP(value) != IS_BOOL) convert_to_boolean(*value); + transport.writeI8(Z_BVAL_PP(value) ? 1 : 0); + return; + case T_BYTE: + if (Z_TYPE_PP(value) != IS_LONG) convert_to_long(*value); + transport.writeI8(Z_LVAL_PP(value)); + return; + case T_I16: + if (Z_TYPE_PP(value) != IS_LONG) convert_to_long(*value); + transport.writeI16(Z_LVAL_PP(value)); + return; + case T_I32: + if (Z_TYPE_PP(value) != IS_LONG) convert_to_long(*value); + transport.writeI32(Z_LVAL_PP(value)); + return; + case T_I64: + case T_U64: + if (Z_TYPE_PP(value) != IS_LONG) convert_to_long(*value); + transport.writeI64(Z_LVAL_PP(value)); + return; + case T_DOUBLE: { + union { + int64_t c; + double d; + } a; + if (Z_TYPE_PP(value) != IS_DOUBLE) convert_to_double(*value); + a.d = Z_DVAL_PP(value); + transport.writeI64(a.c); + } return; + //case T_UTF7: + case T_UTF8: + case T_UTF16: + case T_STRING: + if (Z_TYPE_PP(value) != IS_STRING) convert_to_string(*value); + transport.writeString(Z_STRVAL_PP(value), Z_STRLEN_PP(value)); + return; + case T_MAP: { + if (Z_TYPE_PP(value) != IS_ARRAY) convert_to_array(*value); + if (Z_TYPE_PP(value) != IS_ARRAY) { + throw_tprotocolexception("Attempt to send an incompatible type as an array (T_MAP)", INVALID_DATA); + } + HashTable* ht = Z_ARRVAL_PP(value); + zval** val_ptr; + + zend_hash_find(fieldspec, "ktype", 6, (void**)&val_ptr); + if (Z_TYPE_PP(val_ptr) != IS_LONG) convert_to_long(*val_ptr); + uint8_t keytype = Z_LVAL_PP(val_ptr); + transport.writeI8(keytype); + zend_hash_find(fieldspec, "vtype", 6, (void**)&val_ptr); + if (Z_TYPE_PP(val_ptr) != IS_LONG) convert_to_long(*val_ptr); + uint8_t valtype = Z_LVAL_PP(val_ptr); + transport.writeI8(valtype); + + zend_hash_find(fieldspec, "val", 4, (void**)&val_ptr); + HashTable* valspec = Z_ARRVAL_PP(val_ptr); + + transport.writeI32(zend_hash_num_elements(ht)); + HashPosition key_ptr; + for (zend_hash_internal_pointer_reset_ex(ht, &key_ptr); zend_hash_get_current_data_ex(ht, (void**)&val_ptr, &key_ptr) == SUCCESS; zend_hash_move_forward_ex(ht, &key_ptr)) { + binary_serialize_hashtable_key(keytype, transport, ht, key_ptr); + binary_serialize(valtype, transport, val_ptr, valspec); + } + } return; + case T_LIST: { + if (Z_TYPE_PP(value) != IS_ARRAY) convert_to_array(*value); + if (Z_TYPE_PP(value) != IS_ARRAY) { + throw_tprotocolexception("Attempt to send an incompatible type as an array (T_LIST)", INVALID_DATA); + } + HashTable* ht = Z_ARRVAL_PP(value); + zval** val_ptr; + + zend_hash_find(fieldspec, "etype", 6, (void**)&val_ptr); + if (Z_TYPE_PP(val_ptr) != IS_LONG) convert_to_long(*val_ptr); + uint8_t valtype = Z_LVAL_PP(val_ptr); + transport.writeI8(valtype); + + zend_hash_find(fieldspec, "elem", 5, (void**)&val_ptr); + HashTable* valspec = Z_ARRVAL_PP(val_ptr); + + transport.writeI32(zend_hash_num_elements(ht)); + HashPosition key_ptr; + for (zend_hash_internal_pointer_reset_ex(ht, &key_ptr); zend_hash_get_current_data_ex(ht, (void**)&val_ptr, &key_ptr) == SUCCESS; zend_hash_move_forward_ex(ht, &key_ptr)) { + binary_serialize(valtype, transport, val_ptr, valspec); + } + } return; + case T_SET: { + if (Z_TYPE_PP(value) != IS_ARRAY) convert_to_array(*value); + if (Z_TYPE_PP(value) != IS_ARRAY) { + throw_tprotocolexception("Attempt to send an incompatible type as an array (T_SET)", INVALID_DATA); + } + HashTable* ht = Z_ARRVAL_PP(value); + zval** val_ptr; + + zend_hash_find(fieldspec, "etype", 6, (void**)&val_ptr); + if (Z_TYPE_PP(val_ptr) != IS_LONG) convert_to_long(*val_ptr); + uint8_t keytype = Z_LVAL_PP(val_ptr); + transport.writeI8(keytype); + + transport.writeI32(zend_hash_num_elements(ht)); + HashPosition key_ptr; + for (zend_hash_internal_pointer_reset_ex(ht, &key_ptr); zend_hash_get_current_data_ex(ht, (void**)&val_ptr, &key_ptr) == SUCCESS; zend_hash_move_forward_ex(ht, &key_ptr)) { + binary_serialize_hashtable_key(keytype, transport, ht, key_ptr); + } + } return; + }; + char errbuf[128]; + sprintf(errbuf, "Unknown thrift typeID %d", thrift_typeID); + throw_tprotocolexception(errbuf, INVALID_DATA); +} + + +void binary_serialize_spec(zval* zthis, PHPOutputTransport& transport, HashTable* spec) { + HashPosition key_ptr; + zval** val_ptr; + + TSRMLS_FETCH(); + zend_class_entry* ce = zend_get_class_entry(zthis TSRMLS_CC); + + for (zend_hash_internal_pointer_reset_ex(spec, &key_ptr); zend_hash_get_current_data_ex(spec, (void**)&val_ptr, &key_ptr) == SUCCESS; zend_hash_move_forward_ex(spec, &key_ptr)) { + ulong fieldno; + if (zend_hash_get_current_key_ex(spec, NULL, NULL, &fieldno, 0, &key_ptr) != HASH_KEY_IS_LONG) { + throw_tprotocolexception("Bad keytype in TSPEC (expected 'long')", INVALID_DATA); + return; + } + HashTable* fieldspec = Z_ARRVAL_PP(val_ptr); + + // field name + zend_hash_find(fieldspec, "var", 4, (void**)&val_ptr); + char* varname = Z_STRVAL_PP(val_ptr); + + // thrift type + zend_hash_find(fieldspec, "type", 5, (void**)&val_ptr); + if (Z_TYPE_PP(val_ptr) != IS_LONG) convert_to_long(*val_ptr); + int8_t ttype = Z_LVAL_PP(val_ptr); + + zval* prop = zend_read_property(ce, zthis, varname, strlen(varname), false TSRMLS_CC); + if (Z_TYPE_P(prop) != IS_NULL) { + transport.writeI8(ttype); + transport.writeI16(fieldno); + binary_serialize(ttype, transport, &prop, fieldspec); + } + } + transport.writeI8(T_STOP); // struct end +} + +// 6 params: $transport $method_name $ttype $request_struct $seqID $strict_write +PHP_FUNCTION(thrift_protocol_write_binary) { + int argc = ZEND_NUM_ARGS(); + if (argc < 6) { + WRONG_PARAM_COUNT; + } + + zval ***args = (zval***) emalloc(argc * sizeof(zval**)); + zend_get_parameters_array_ex(argc, args); + + if (Z_TYPE_PP(args[0]) != IS_OBJECT) { + php_error_docref(NULL TSRMLS_CC, E_ERROR, "1st parameter is not an object (transport)"); + efree(args); + RETURN_NULL(); + } + + if (Z_TYPE_PP(args[1]) != IS_STRING) { + php_error_docref(NULL TSRMLS_CC, E_ERROR, "2nd parameter is not a string (method name)"); + efree(args); + RETURN_NULL(); + } + + if (Z_TYPE_PP(args[3]) != IS_OBJECT) { + php_error_docref(NULL TSRMLS_CC, E_ERROR, "4th parameter is not an object (request struct)"); + efree(args); + RETURN_NULL(); + } + + PHPOutputTransport transport(*args[0]); + const char* method_name = Z_STRVAL_PP(args[1]); + convert_to_long(*args[2]); + int32_t msgtype = Z_LVAL_PP(args[2]); + zval* request_struct = *args[3]; + convert_to_long(*args[4]); + int32_t seqID = Z_LVAL_PP(args[4]); + convert_to_boolean(*args[5]); + bool strictWrite = Z_BVAL_PP(args[5]); + efree(args); + args = NULL; + + try { + if (strictWrite) { + int32_t version = VERSION_1 | msgtype; + transport.writeI32(version); + transport.writeString(method_name, strlen(method_name)); + transport.writeI32(seqID); + } else { + transport.writeString(method_name, strlen(method_name)); + transport.writeI8(msgtype); + transport.writeI32(seqID); + } + + zval* spec = zend_read_static_property(zend_get_class_entry(request_struct TSRMLS_CC), "_TSPEC", 6, false TSRMLS_CC); + binary_serialize_spec(request_struct, transport, Z_ARRVAL_P(spec)); + } catch (const PHPExceptionWrapper& ex) { + zend_throw_exception_object(ex TSRMLS_CC); + RETURN_NULL(); + } +} + +// 3 params: $transport $response_Typename $strict_read +PHP_FUNCTION(thrift_protocol_read_binary) { + int argc = ZEND_NUM_ARGS(); + + if (argc < 3) { + WRONG_PARAM_COUNT; + } + + zval ***args = (zval***) emalloc(argc * sizeof(zval**)); + zend_get_parameters_array_ex(argc, args); + + if (Z_TYPE_PP(args[0]) != IS_OBJECT) { + php_error_docref(NULL TSRMLS_CC, E_ERROR, "1st parameter is not an object (transport)"); + efree(args); + RETURN_NULL(); + } + + if (Z_TYPE_PP(args[1]) != IS_STRING) { + php_error_docref(NULL TSRMLS_CC, E_ERROR, "2nd parameter is not a string (typename of expected response struct)"); + efree(args); + RETURN_NULL(); + } + + PHPInputTransport transport(*args[0]); + char* obj_typename = Z_STRVAL_PP(args[1]); + convert_to_boolean(*args[2]); + bool strict_read = Z_BVAL_PP(args[2]); + efree(args); + args = NULL; + + try { + int8_t messageType = 0; + int32_t sz = transport.readI32(); + + if (sz < 0) { + // Check for correct version number + int32_t version = sz & VERSION_MASK; + if (version != VERSION_1) { + throw_tprotocolexception("Bad version identifier", BAD_VERSION); + } + messageType = (sz & 0x000000ff); + int32_t namelen = transport.readI32(); + // skip the name string and the sequence ID, we don't care about those + transport.skip(namelen + 4); + } else { + if (strict_read) { + throw_tprotocolexception("No version identifier... old protocol client in strict mode?", BAD_VERSION); + } else { + // Handle pre-versioned input + transport.skip(sz); // skip string body + messageType = transport.readI8(); + transport.skip(4); // skip sequence number + } + } + + if (messageType == T_EXCEPTION) { + zval* ex; + MAKE_STD_ZVAL(ex); + createObject("TApplicationException", ex); + zval* spec = zend_read_static_property(zend_get_class_entry(ex TSRMLS_CC), "_TSPEC", 6, false TSRMLS_CC); + binary_deserialize_spec(ex, transport, Z_ARRVAL_P(spec)); + throw PHPExceptionWrapper(ex); + } + + createObject(obj_typename, return_value); + zval* spec = zend_read_static_property(zend_get_class_entry(return_value TSRMLS_CC), "_TSPEC", 6, false TSRMLS_CC); + binary_deserialize_spec(return_value, transport, Z_ARRVAL_P(spec)); + } catch (const PHPExceptionWrapper& ex) { + zend_throw_exception_object(ex TSRMLS_CC); + RETURN_NULL(); + } +} + diff --git a/lib/php/src/ext/thrift_protocol/php_thrift_protocol.h b/lib/php/src/ext/thrift_protocol/php_thrift_protocol.h new file mode 100644 index 00000000..c9a3e00f --- /dev/null +++ b/lib/php/src/ext/thrift_protocol/php_thrift_protocol.h @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +PHP_FUNCTION(thrift_protocol_write_binary); +PHP_FUNCTION(thrift_protocol_read_binary); + +extern zend_module_entry thrift_protocole_module_entry; + diff --git a/lib/php/src/protocol/TBinaryProtocol.php b/lib/php/src/protocol/TBinaryProtocol.php new file mode 100644 index 00000000..31bbbf9d --- /dev/null +++ b/lib/php/src/protocol/TBinaryProtocol.php @@ -0,0 +1,431 @@ +strictRead_ = $strictRead; + $this->strictWrite_ = $strictWrite; + } + + public function writeMessageBegin($name, $type, $seqid) { + if ($this->strictWrite_) { + $version = self::VERSION_1 | $type; + return + $this->writeI32($version) + + $this->writeString($name) + + $this->writeI32($seqid); + } else { + return + $this->writeString($name) + + $this->writeByte($type) + + $this->writeI32($seqid); + } + } + + public function writeMessageEnd() { + return 0; + } + + public function writeStructBegin($name) { + return 0; + } + + public function writeStructEnd() { + return 0; + } + + public function writeFieldBegin($fieldName, $fieldType, $fieldId) { + return + $this->writeByte($fieldType) + + $this->writeI16($fieldId); + } + + public function writeFieldEnd() { + return 0; + } + + public function writeFieldStop() { + return + $this->writeByte(TType::STOP); + } + + public function writeMapBegin($keyType, $valType, $size) { + return + $this->writeByte($keyType) + + $this->writeByte($valType) + + $this->writeI32($size); + } + + public function writeMapEnd() { + return 0; + } + + public function writeListBegin($elemType, $size) { + return + $this->writeByte($elemType) + + $this->writeI32($size); + } + + public function writeListEnd() { + return 0; + } + + public function writeSetBegin($elemType, $size) { + return + $this->writeByte($elemType) + + $this->writeI32($size); + } + + public function writeSetEnd() { + return 0; + } + + public function writeBool($value) { + $data = pack('c', $value ? 1 : 0); + $this->trans_->write($data, 1); + return 1; + } + + public function writeByte($value) { + $data = pack('c', $value); + $this->trans_->write($data, 1); + return 1; + } + + public function writeI16($value) { + $data = pack('n', $value); + $this->trans_->write($data, 2); + return 2; + } + + public function writeI32($value) { + $data = pack('N', $value); + $this->trans_->write($data, 4); + return 4; + } + + public function writeI64($value) { + // If we are on a 32bit architecture we have to explicitly deal with + // 64-bit twos-complement arithmetic since PHP wants to treat all ints + // as signed and any int over 2^31 - 1 as a float + if (PHP_INT_SIZE == 4) { + $neg = $value < 0; + + if ($neg) { + $value *= -1; + } + + $hi = (int)($value / 4294967296); + $lo = (int)$value; + + if ($neg) { + $hi = ~$hi; + $lo = ~$lo; + if (($lo & (int)0xffffffff) == (int)0xffffffff) { + $lo = 0; + $hi++; + } else { + $lo++; + } + } + $data = pack('N2', $hi, $lo); + + } else { + $hi = $value >> 32; + $lo = $value & 0xFFFFFFFF; + $data = pack('N2', $hi, $lo); + } + + $this->trans_->write($data, 8); + return 8; + } + + public function writeDouble($value) { + $data = pack('d', $value); + $this->trans_->write(strrev($data), 8); + return 8; + } + + public function writeString($value) { + $len = strlen($value); + $result = $this->writeI32($len); + if ($len) { + $this->trans_->write($value, $len); + } + return $result + $len; + } + + public function readMessageBegin(&$name, &$type, &$seqid) { + $result = $this->readI32($sz); + if ($sz < 0) { + $version = (int) ($sz & self::VERSION_MASK); + if ($version != (int) self::VERSION_1) { + throw new TProtocolException('Bad version identifier: '.$sz, TProtocolException::BAD_VERSION); + } + $type = $sz & 0x000000ff; + $result += + $this->readString($name) + + $this->readI32($seqid); + } else { + if ($this->strictRead_) { + throw new TProtocolException('No version identifier, old protocol client?', TProtocolException::BAD_VERSION); + } else { + // Handle pre-versioned input + $name = $this->trans_->readAll($sz); + $result += + $sz + + $this->readByte($type) + + $this->readI32($seqid); + } + } + return $result; + } + + public function readMessageEnd() { + return 0; + } + + public function readStructBegin(&$name) { + $name = ''; + return 0; + } + + public function readStructEnd() { + return 0; + } + + public function readFieldBegin(&$name, &$fieldType, &$fieldId) { + $result = $this->readByte($fieldType); + if ($fieldType == TType::STOP) { + $fieldId = 0; + return $result; + } + $result += $this->readI16($fieldId); + return $result; + } + + public function readFieldEnd() { + return 0; + } + + public function readMapBegin(&$keyType, &$valType, &$size) { + return + $this->readByte($keyType) + + $this->readByte($valType) + + $this->readI32($size); + } + + public function readMapEnd() { + return 0; + } + + public function readListBegin(&$elemType, &$size) { + return + $this->readByte($elemType) + + $this->readI32($size); + } + + public function readListEnd() { + return 0; + } + + public function readSetBegin(&$elemType, &$size) { + return + $this->readByte($elemType) + + $this->readI32($size); + } + + public function readSetEnd() { + return 0; + } + + public function readBool(&$value) { + $data = $this->trans_->readAll(1); + $arr = unpack('c', $data); + $value = $arr[1] == 1; + return 1; + } + + public function readByte(&$value) { + $data = $this->trans_->readAll(1); + $arr = unpack('c', $data); + $value = $arr[1]; + return 1; + } + + public function readI16(&$value) { + $data = $this->trans_->readAll(2); + $arr = unpack('n', $data); + $value = $arr[1]; + if ($value > 0x7fff) { + $value = 0 - (($value - 1) ^ 0xffff); + } + return 2; + } + + public function readI32(&$value) { + $data = $this->trans_->readAll(4); + $arr = unpack('N', $data); + $value = $arr[1]; + if ($value > 0x7fffffff) { + $value = 0 - (($value - 1) ^ 0xffffffff); + } + return 4; + } + + public function readI64(&$value) { + $data = $this->trans_->readAll(8); + + $arr = unpack('N2', $data); + + // If we are on a 32bit architecture we have to explicitly deal with + // 64-bit twos-complement arithmetic since PHP wants to treat all ints + // as signed and any int over 2^31 - 1 as a float + if (PHP_INT_SIZE == 4) { + + $hi = $arr[1]; + $lo = $arr[2]; + $isNeg = $hi < 0; + + // Check for a negative + if ($isNeg) { + $hi = ~$hi & (int)0xffffffff; + $lo = ~$lo & (int)0xffffffff; + + if ($lo == (int)0xffffffff) { + $hi++; + $lo = 0; + } else { + $lo++; + } + } + + // Force 32bit words in excess of 2G to pe positive - we deal wigh sign + // explicitly below + + if ($hi & (int)0x80000000) { + $hi &= (int)0x7fffffff; + $hi += 0x80000000; + } + + if ($lo & (int)0x80000000) { + $lo &= (int)0x7fffffff; + $lo += 0x80000000; + } + + $value = $hi * 4294967296 + $lo; + + if ($isNeg) { + $value = 0 - $value; + } + } else { + + // Upcast negatives in LSB bit + if ($arr[2] & 0x80000000) { + $arr[2] = $arr[2] & 0xffffffff; + } + + // Check for a negative + if ($arr[1] & 0x80000000) { + $arr[1] = $arr[1] & 0xffffffff; + $arr[1] = $arr[1] ^ 0xffffffff; + $arr[2] = $arr[2] ^ 0xffffffff; + $value = 0 - $arr[1]*4294967296 - $arr[2] - 1; + } else { + $value = $arr[1]*4294967296 + $arr[2]; + } + } + + return 8; + } + + public function readDouble(&$value) { + $data = strrev($this->trans_->readAll(8)); + $arr = unpack('d', $data); + $value = $arr[1]; + return 8; + } + + public function readString(&$value) { + $result = $this->readI32($len); + if ($len) { + $value = $this->trans_->readAll($len); + } else { + $value = ''; + } + return $result + $len; + } +} + +/** + * Binary Protocol Factory + */ +class TBinaryProtocolFactory implements TProtocolFactory { + private $strictRead_ = false; + private $strictWrite_ = false; + + public function __construct($strictRead=false, $strictWrite=false) { + $this->strictRead_ = $strictRead; + $this->strictWrite_ = $strictWrite; + } + + public function getProtocol($trans) { + return new TBinaryProtocol($trans, $this->strictRead, $this->strictWrite); + } +} + +/** + * Accelerated binary protocol: used in conjunction with the thrift_protocol + * extension for faster deserialization + */ +class TBinaryProtocolAccelerated extends TBinaryProtocol { + public function __construct($trans, $strictRead=false, $strictWrite=true) { + // If the transport doesn't implement putBack, wrap it in a + // TBufferedTransport (which does) + if (!method_exists($trans, 'putBack')) { + $trans = new TBufferedTransport($trans); + } + parent::__construct($trans, $strictRead, $strictWrite); + } + public function isStrictRead() { + return $this->strictRead_; + } + public function isStrictWrite() { + return $this->strictWrite_; + } +} + +?> diff --git a/lib/php/src/protocol/TProtocol.php b/lib/php/src/protocol/TProtocol.php new file mode 100644 index 00000000..e9ff41a3 --- /dev/null +++ b/lib/php/src/protocol/TProtocol.php @@ -0,0 +1,377 @@ +trans_ = $trans; + } + + /** + * Accessor for transport + * + * @return TTransport + */ + public function getTransport() { + return $this->trans_; + } + + /** + * Writes the message header + * + * @param string $name Function name + * @param int $type message type TMessageType::CALL or TMessageType::REPLY + * @param int $seqid The sequence id of this message + */ + public abstract function writeMessageBegin($name, $type, $seqid); + + /** + * Close the message + */ + public abstract function writeMessageEnd(); + + /** + * Writes a struct header. + * + * @param string $name Struct name + * @throws TException on write error + * @return int How many bytes written + */ + public abstract function writeStructBegin($name); + + /** + * Close a struct. + * + * @throws TException on write error + * @return int How many bytes written + */ + public abstract function writeStructEnd(); + + /* + * Starts a field. + * + * @param string $name Field name + * @param int $type Field type + * @param int $fid Field id + * @throws TException on write error + * @return int How many bytes written + */ + public abstract function writeFieldBegin($fieldName, $fieldType, $fieldId); + + public abstract function writeFieldEnd(); + + public abstract function writeFieldStop(); + + public abstract function writeMapBegin($keyType, $valType, $size); + + public abstract function writeMapEnd(); + + public abstract function writeListBegin($elemType, $size); + + public abstract function writeListEnd(); + + public abstract function writeSetBegin($elemType, $size); + + public abstract function writeSetEnd(); + + public abstract function writeBool($bool); + + public abstract function writeByte($byte); + + public abstract function writeI16($i16); + + public abstract function writeI32($i32); + + public abstract function writeI64($i64); + + public abstract function writeDouble($dub); + + public abstract function writeString($str); + + /** + * Reads the message header + * + * @param string $name Function name + * @param int $type message type TMessageType::CALL or TMessageType::REPLY + * @parem int $seqid The sequence id of this message + */ + public abstract function readMessageBegin(&$name, &$type, &$seqid); + + /** + * Read the close of message + */ + public abstract function readMessageEnd(); + + public abstract function readStructBegin(&$name); + + public abstract function readStructEnd(); + + public abstract function readFieldBegin(&$name, &$fieldType, &$fieldId); + + public abstract function readFieldEnd(); + + public abstract function readMapBegin(&$keyType, &$valType, &$size); + + public abstract function readMapEnd(); + + public abstract function readListBegin(&$elemType, &$size); + + public abstract function readListEnd(); + + public abstract function readSetBegin(&$elemType, &$size); + + public abstract function readSetEnd(); + + public abstract function readBool(&$bool); + + public abstract function readByte(&$byte); + + public abstract function readI16(&$i16); + + public abstract function readI32(&$i32); + + public abstract function readI64(&$i64); + + public abstract function readDouble(&$dub); + + public abstract function readString(&$str); + + /** + * The skip function is a utility to parse over unrecognized date without + * causing corruption. + * + * @param TType $type What type is it + */ + public function skip($type) { + switch ($type) { + case TType::BOOL: + return $this->readBool($bool); + case TType::BYTE: + return $this->readByte($byte); + case TType::I16: + return $this->readI16($i16); + case TType::I32: + return $this->readI32($i32); + case TType::I64: + return $this->readI64($i64); + case TType::DOUBLE: + return $this->readDouble($dub); + case TType::STRING: + return $this->readString($str); + case TType::STRUCT: + { + $result = $this->readStructBegin($name); + while (true) { + $result += $this->readFieldBegin($name, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + $result += $this->skip($ftype); + $result += $this->readFieldEnd(); + } + $result += $this->readStructEnd(); + return $result; + } + case TType::MAP: + { + $result = $this->readMapBegin($keyType, $valType, $size); + for ($i = 0; $i < $size; $i++) { + $result += $this->skip($keyType); + $result += $this->skip($valType); + } + $result += $this->readMapEnd(); + return $result; + } + case TType::SET: + { + $result = $this->readSetBegin($elemType, $size); + for ($i = 0; $i < $size; $i++) { + $result += $this->skip($elemType); + } + $result += $this->readSetEnd(); + return $result; + } + case TType::LST: + { + $result = $this->readListBegin($elemType, $size); + for ($i = 0; $i < $size; $i++) { + $result += $this->skip($elemType); + } + $result += $this->readListEnd(); + return $result; + } + default: + return 0; + } + } + + /** + * Utility for skipping binary data + * + * @param TTransport $itrans TTransport object + * @param int $type Field type + */ + public static function skipBinary($itrans, $type) { + switch ($type) { + case TType::BOOL: + return $itrans->readAll(1); + case TType::BYTE: + return $itrans->readAll(1); + case TType::I16: + return $itrans->readAll(2); + case TType::I32: + return $itrans->readAll(4); + case TType::I64: + return $itrans->readAll(8); + case TType::DOUBLE: + return $itrans->readAll(8); + case TType::STRING: + $len = unpack('N', $itrans->readAll(4)); + $len = $len[1]; + if ($len > 0x7fffffff) { + $len = 0 - (($len - 1) ^ 0xffffffff); + } + return 4 + $itrans->readAll($len); + case TType::STRUCT: + { + $result = 0; + while (true) { + $ftype = 0; + $fid = 0; + $data = $itrans->readAll(1); + $arr = unpack('c', $data); + $ftype = $arr[1]; + if ($ftype == TType::STOP) { + break; + } + // I16 field id + $result += $itrans->readAll(2); + $result += self::skipBinary($itrans, $ftype); + } + return $result; + } + case TType::MAP: + { + // Ktype + $data = $itrans->readAll(1); + $arr = unpack('c', $data); + $ktype = $arr[1]; + // Vtype + $data = $itrans->readAll(1); + $arr = unpack('c', $data); + $vtype = $arr[1]; + // Size + $data = $itrans->readAll(4); + $arr = unpack('N', $data); + $size = $arr[1]; + if ($size > 0x7fffffff) { + $size = 0 - (($size - 1) ^ 0xffffffff); + } + $result = 6; + for ($i = 0; $i < $size; $i++) { + $result += self::skipBinary($itrans, $ktype); + $result += self::skipBinary($itrans, $vtype); + } + return $result; + } + case TType::SET: + case TType::LST: + { + // Vtype + $data = $itrans->readAll(1); + $arr = unpack('c', $data); + $vtype = $arr[1]; + // Size + $data = $itrans->readAll(4); + $arr = unpack('N', $data); + $size = $arr[1]; + if ($size > 0x7fffffff) { + $size = 0 - (($size - 1) ^ 0xffffffff); + } + $result = 5; + for ($i = 0; $i < $size; $i++) { + $result += self::skipBinary($itrans, $vtype); + } + return $result; + } + default: + return 0; + } + } +} + +/** + * Protocol factory creates protocol objects from transports + */ +interface TProtocolFactory { + /** + * Build a protocol from the base transport + * + * @return TProtcol protocol + */ + public function getProtocol($trans); +} + + +?> diff --git a/lib/php/src/transport/TBufferedTransport.php b/lib/php/src/transport/TBufferedTransport.php new file mode 100644 index 00000000..cfae767e --- /dev/null +++ b/lib/php/src/transport/TBufferedTransport.php @@ -0,0 +1,163 @@ +transport_ = $transport; + $this->rBufSize_ = $rBufSize; + $this->wBufSize_ = $wBufSize; + } + + /** + * The underlying transport + * + * @var TTransport + */ + protected $transport_ = null; + + /** + * The receive buffer size + * + * @var int + */ + protected $rBufSize_ = 512; + + /** + * The write buffer size + * + * @var int + */ + protected $wBufSize_ = 512; + + /** + * The write buffer. + * + * @var string + */ + protected $wBuf_ = ''; + + /** + * The read buffer. + * + * @var string + */ + protected $rBuf_ = ''; + + public function isOpen() { + return $this->transport_->isOpen(); + } + + public function open() { + $this->transport_->open(); + } + + public function close() { + $this->transport_->close(); + } + + public function putBack($data) { + if (strlen($this->rBuf_) === 0) { + $this->rBuf_ = $data; + } else { + $this->rBuf_ = ($data . $this->rBuf_); + } + } + + /** + * The reason that we customize readAll here is that the majority of PHP + * streams are already internally buffered by PHP. The socket stream, for + * example, buffers internally and blocks if you call read with $len greater + * than the amount of data available, unlike recv() in C. + * + * Therefore, use the readAll method of the wrapped transport inside + * the buffered readAll. + */ + public function readAll($len) { + $have = strlen($this->rBuf_); + if ($have == 0) { + $data = $this->transport_->readAll($len); + } else if ($have < $len) { + $data = $this->rBuf_; + $this->rBuf_ = ''; + $data .= $this->transport_->readAll($len - $have); + } else if ($have == $len) { + $data = $this->rBuf_; + $this->rBuf_ = ''; + } else if ($have > $len) { + $data = substr($this->rBuf_, 0, $len); + $this->rBuf_ = substr($this->rBuf_, $len); + } + return $data; + } + + public function read($len) { + if (strlen($this->rBuf_) === 0) { + $this->rBuf_ = $this->transport_->read($this->rBufSize_); + } + + if (strlen($this->rBuf_) <= $len) { + $ret = $this->rBuf_; + $this->rBuf_ = ''; + return $ret; + } + + $ret = substr($this->rBuf_, 0, $len); + $this->rBuf_ = substr($this->rBuf_, $len); + return $ret; + } + + public function write($buf) { + $this->wBuf_ .= $buf; + if (strlen($this->wBuf_) >= $this->wBufSize_) { + $out = $this->wBuf_; + + // Note that we clear the internal wBuf_ prior to the underlying write + // to ensure we're in a sane state (i.e. internal buffer cleaned) + // if the underlying write throws up an exception + $this->wBuf_ = ''; + $this->transport_->write($out); + } + } + + public function flush() { + if (strlen($this->wBuf_) > 0) { + $this->transport_->write($this->wBuf_); + $this->wBuf_ = ''; + } + $this->transport_->flush(); + } + +} + +?> diff --git a/lib/php/src/transport/TFramedTransport.php b/lib/php/src/transport/TFramedTransport.php new file mode 100644 index 00000000..dc57392f --- /dev/null +++ b/lib/php/src/transport/TFramedTransport.php @@ -0,0 +1,179 @@ +transport_ = $transport; + $this->read_ = $read; + $this->write_ = $write; + } + + public function isOpen() { + return $this->transport_->isOpen(); + } + + public function open() { + $this->transport_->open(); + } + + public function close() { + $this->transport_->close(); + } + + /** + * Reads from the buffer. When more data is required reads another entire + * chunk and serves future reads out of that. + * + * @param int $len How much data + */ + public function read($len) { + if (!$this->read_) { + return $this->transport_->read($len); + } + + if (strlen($this->rBuf_) === 0) { + $this->readFrame(); + } + + // Just return full buff + if ($len >= strlen($this->rBuf_)) { + $out = $this->rBuf_; + $this->rBuf_ = null; + return $out; + } + + // Return substr + $out = substr($this->rBuf_, 0, $len); + $this->rBuf_ = substr($this->rBuf_, $len); + return $out; + } + + /** + * Put previously read data back into the buffer + * + * @param string $data data to return + */ + public function putBack($data) { + if (strlen($this->rBuf_) === 0) { + $this->rBuf_ = $data; + } else { + $this->rBuf_ = ($data . $this->rBuf_); + } + } + + /** + * Reads a chunk of data into the internal read buffer. + */ + private function readFrame() { + $buf = $this->transport_->readAll(4); + $val = unpack('N', $buf); + $sz = $val[1]; + + $this->rBuf_ = $this->transport_->readAll($sz); + } + + /** + * Writes some data to the pending output buffer. + * + * @param string $buf The data + * @param int $len Limit of bytes to write + */ + public function write($buf, $len=null) { + if (!$this->write_) { + return $this->transport_->write($buf, $len); + } + + if ($len !== null && $len < strlen($buf)) { + $buf = substr($buf, 0, $len); + } + $this->wBuf_ .= $buf; + } + + /** + * Writes the output buffer to the stream in the format of a 4-byte length + * followed by the actual data. + */ + public function flush() { + if (!$this->write_) { + return $this->transport_->flush(); + } + + $out = pack('N', strlen($this->wBuf_)); + $out .= $this->wBuf_; + + // Note that we clear the internal wBuf_ prior to the underlying write + // to ensure we're in a sane state (i.e. internal buffer cleaned) + // if the underlying write throws up an exception + $this->wBuf_ = ''; + $this->transport_->write($out); + $this->transport_->flush(); + } + +} diff --git a/lib/php/src/transport/THttpClient.php b/lib/php/src/transport/THttpClient.php new file mode 100644 index 00000000..224d403b --- /dev/null +++ b/lib/php/src/transport/THttpClient.php @@ -0,0 +1,202 @@ + 0) && ($uri{0} != '/')) { + $uri = '/'.$uri; + } + $this->scheme_ = $scheme; + $this->host_ = $host; + $this->port_ = $port; + $this->uri_ = $uri; + $this->buf_ = ''; + $this->handle_ = null; + $this->timeout_ = null; + } + + /** + * Set read timeout + * + * @param float $timeout + */ + public function setTimeoutSecs($timeout) { + $this->timeout_ = $timeout; + } + + /** + * Whether this transport is open. + * + * @return boolean true if open + */ + public function isOpen() { + return true; + } + + /** + * Open the transport for reading/writing + * + * @throws TTransportException if cannot open + */ + public function open() {} + + /** + * Close the transport. + */ + public function close() { + if ($this->handle_) { + @fclose($this->handle_); + $this->handle_ = null; + } + } + + /** + * Read some data into the array. + * + * @param int $len How much to read + * @return string The data that has been read + * @throws TTransportException if cannot read any more data + */ + public function read($len) { + $data = @fread($this->handle_, $len); + if ($data === FALSE || $data === '') { + $md = stream_get_meta_data($this->handle_); + if ($md['timed_out']) { + throw new TTransportException('THttpClient: timed out reading '.$len.' bytes from '.$this->host_.':'.$this->port_.'/'.$this->uri_, TTransportException::TIMED_OUT); + } else { + throw new TTransportException('THttpClient: Could not read '.$len.' bytes from '.$this->host_.':'.$this->port_.'/'.$this->uri_, TTransportException::UNKNOWN); + } + } + return $data; + } + + /** + * Writes some data into the pending buffer + * + * @param string $buf The data to write + * @throws TTransportException if writing fails + */ + public function write($buf) { + $this->buf_ .= $buf; + } + + /** + * Opens and sends the actual request over the HTTP connection + * + * @throws TTransportException if a writing error occurs + */ + public function flush() { + // God, PHP really has some esoteric ways of doing simple things. + $host = $this->host_.($this->port_ != 80 ? ':'.$this->port_ : ''); + + $headers = array('Host: '.$host, + 'Accept: application/x-thrift', + 'User-Agent: PHP/THttpClient', + 'Content-Type: application/x-thrift', + 'Content-Length: '.strlen($this->buf_)); + + $options = array('method' => 'POST', + 'header' => implode("\r\n", $headers), + 'max_redirects' => 1, + 'content' => $this->buf_); + if ($this->timeout_ > 0) { + $options['timeout'] = $this->timeout_; + } + $this->buf_ = ''; + + $contextid = stream_context_create(array('http' => $options)); + $this->handle_ = @fopen($this->scheme_.'://'.$host.$this->uri_, 'r', false, $contextid); + + // Connect failed? + if ($this->handle_ === FALSE) { + $this->handle_ = null; + $error = 'THttpClient: Could not connect to '.$host.$this->uri_; + throw new TTransportException($error, TTransportException::NOT_OPEN); + } + } + +} + +?> diff --git a/lib/php/src/transport/TMemoryBuffer.php b/lib/php/src/transport/TMemoryBuffer.php new file mode 100644 index 00000000..01eb0f5a --- /dev/null +++ b/lib/php/src/transport/TMemoryBuffer.php @@ -0,0 +1,84 @@ +buf_ = $buf; + } + + protected $buf_ = ''; + + public function isOpen() { + return true; + } + + public function open() {} + + public function close() {} + + public function write($buf) { + $this->buf_ .= $buf; + } + + public function read($len) { + if (strlen($this->buf_) === 0) { + throw new TTransportException('TMemoryBuffer: Could not read ' . + $len . ' bytes from buffer.', + TTransportException::UNKNOWN); + } + + if (strlen($this->buf_) <= $len) { + $ret = $this->buf_; + $this->buf_ = ''; + return $ret; + } + + $ret = substr($this->buf_, 0, $len); + $this->buf_ = substr($this->buf_, $len); + + return $ret; + } + + function getBuffer() { + return $this->buf_; + } + + public function available() { + return strlen($this->buf_); + } +} + +?> diff --git a/lib/php/src/transport/TNullTransport.php b/lib/php/src/transport/TNullTransport.php new file mode 100644 index 00000000..bada5dfb --- /dev/null +++ b/lib/php/src/transport/TNullTransport.php @@ -0,0 +1,48 @@ + diff --git a/lib/php/src/transport/TPhpStream.php b/lib/php/src/transport/TPhpStream.php new file mode 100644 index 00000000..3a1c80b8 --- /dev/null +++ b/lib/php/src/transport/TPhpStream.php @@ -0,0 +1,111 @@ +read_ = $mode & self::MODE_R; + $this->write_ = $mode & self::MODE_W; + } + + public function open() { + if ($this->read_) { + $this->inStream_ = @fopen(self::inStreamName(), 'r'); + if (!is_resource($this->inStream_)) { + throw new TException('TPhpStream: Could not open php://input'); + } + } + if ($this->write_) { + $this->outStream_ = @fopen('php://output', 'w'); + if (!is_resource($this->outStream_)) { + throw new TException('TPhpStream: Could not open php://output'); + } + } + } + + public function close() { + if ($this->read_) { + @fclose($this->inStream_); + $this->inStream_ = null; + } + if ($this->write_) { + @fclose($this->outStream_); + $this->outStream_ = null; + } + } + + public function isOpen() { + return + (!$this->read_ || is_resource($this->inStream_)) && + (!$this->write_ || is_resource($this->outStream_)); + } + + public function read($len) { + $data = @fread($this->inStream_, $len); + if ($data === FALSE || $data === '') { + throw new TException('TPhpStream: Could not read '.$len.' bytes'); + } + return $data; + } + + public function write($buf) { + while (strlen($buf) > 0) { + $got = @fwrite($this->outStream_, $buf); + if ($got === 0 || $got === FALSE) { + throw new TException('TPhpStream: Could not write '.strlen($buf).' bytes'); + } + $buf = substr($buf, $got); + } + } + + public function flush() { + @fflush($this->outStream_); + } + + private static function inStreamName() { + if (php_sapi_name() == 'cli') { + return 'php://stdin'; + } + return 'php://input'; + } + +} + +?> diff --git a/lib/php/src/transport/TSocket.php b/lib/php/src/transport/TSocket.php new file mode 100644 index 00000000..ba3a6318 --- /dev/null +++ b/lib/php/src/transport/TSocket.php @@ -0,0 +1,312 @@ +host_ = $host; + $this->port_ = $port; + $this->persist_ = $persist; + $this->debugHandler_ = $debugHandler ? $debugHandler : 'error_log'; + } + + /** + * Sets the send timeout. + * + * @param int $timeout Timeout in milliseconds. + */ + public function setSendTimeout($timeout) { + $this->sendTimeout_ = $timeout; + } + + /** + * Sets the receive timeout. + * + * @param int $timeout Timeout in milliseconds. + */ + public function setRecvTimeout($timeout) { + $this->recvTimeout_ = $timeout; + } + + /** + * Sets debugging output on or off + * + * @param bool $debug + */ + public function setDebug($debug) { + $this->debug_ = $debug; + } + + /** + * Get the host that this socket is connected to + * + * @return string host + */ + public function getHost() { + return $this->host_; + } + + /** + * Get the remote port that this socket is connected to + * + * @return int port + */ + public function getPort() { + return $this->port_; + } + + /** + * Tests whether this is open + * + * @return bool true if the socket is open + */ + public function isOpen() { + return is_resource($this->handle_); + } + + /** + * Connects the socket. + */ + public function open() { + + if ($this->persist_) { + $this->handle_ = @pfsockopen($this->host_, + $this->port_, + $errno, + $errstr, + $this->sendTimeout_/1000.0); + } else { + $this->handle_ = @fsockopen($this->host_, + $this->port_, + $errno, + $errstr, + $this->sendTimeout_/1000.0); + } + + // Connect failed? + if ($this->handle_ === FALSE) { + $error = 'TSocket: Could not connect to '.$this->host_.':'.$this->port_.' ('.$errstr.' ['.$errno.'])'; + if ($this->debug_) { + call_user_func($this->debugHandler_, $error); + } + throw new TException($error); + } + + stream_set_timeout($this->handle_, 0, $this->sendTimeout_*1000); + $this->sendTimeoutSet_ = TRUE; + } + + /** + * Closes the socket. + */ + public function close() { + if (!$this->persist_) { + @fclose($this->handle_); + $this->handle_ = null; + } + } + + /** + * Uses stream get contents to do the reading + * + * @param int $len How many bytes + * @return string Binary data + */ + public function readAll($len) { + if ($this->sendTimeoutSet_) { + stream_set_timeout($this->handle_, 0, $this->recvTimeout_*1000); + $this->sendTimeoutSet_ = FALSE; + } + // This call does not obey stream_set_timeout values! + // $buf = @stream_get_contents($this->handle_, $len); + + $pre = null; + while (TRUE) { + $buf = @fread($this->handle_, $len); + if ($buf === FALSE || $buf === '') { + $md = stream_get_meta_data($this->handle_); + if ($md['timed_out']) { + throw new TException('TSocket: timed out reading '.$len.' bytes from '. + $this->host_.':'.$this->port_); + } else { + throw new TException('TSocket: Could not read '.$len.' bytes from '. + $this->host_.':'.$this->port_); + } + } else if (($sz = strlen($buf)) < $len) { + $md = stream_get_meta_data($this->handle_); + if ($md['timed_out']) { + throw new TException('TSocket: timed out reading '.$len.' bytes from '. + $this->host_.':'.$this->port_); + } else { + $pre .= $buf; + $len -= $sz; + } + } else { + return $pre.$buf; + } + } + } + + /** + * Read from the socket + * + * @param int $len How many bytes + * @return string Binary data + */ + public function read($len) { + if ($this->sendTimeoutSet_) { + stream_set_timeout($this->handle_, 0, $this->recvTimeout_*1000); + $this->sendTimeoutSet_ = FALSE; + } + $data = @fread($this->handle_, $len); + if ($data === FALSE || $data === '') { + $md = stream_get_meta_data($this->handle_); + if ($md['timed_out']) { + throw new TException('TSocket: timed out reading '.$len.' bytes from '. + $this->host_.':'.$this->port_); + } else { + throw new TException('TSocket: Could not read '.$len.' bytes from '. + $this->host_.':'.$this->port_); + } + } + return $data; + } + + /** + * Write to the socket. + * + * @param string $buf The data to write + */ + public function write($buf) { + if (!$this->sendTimeoutSet_) { + stream_set_timeout($this->handle_, 0, $this->sendTimeout_*1000); + $this->sendTimeoutSet_ = TRUE; + } + while (strlen($buf) > 0) { + $got = @fwrite($this->handle_, $buf); + if ($got === 0 || $got === FALSE) { + $md = stream_get_meta_data($this->handle_); + if ($md['timed_out']) { + throw new TException('TSocket: timed out writing '.strlen($buf).' bytes from '. + $this->host_.':'.$this->port_); + } else { + throw new TException('TSocket: Could not write '.strlen($buf).' bytes '. + $this->host_.':'.$this->port_); + } + } + $buf = substr($buf, $got); + } + } + + /** + * Flush output to the socket. + */ + public function flush() { + $ret = fflush($this->handle_); + if ($ret === FALSE) { + throw new TException('TSocket: Could not flush: '. + $this->host_.':'.$this->port_); + } + } +} + +?> diff --git a/lib/php/src/transport/TSocketPool.php b/lib/php/src/transport/TSocketPool.php new file mode 100644 index 00000000..7f1157cb --- /dev/null +++ b/lib/php/src/transport/TSocketPool.php @@ -0,0 +1,296 @@ + $val) { + $ports[$key] = $port; + } + } + + foreach ($hosts as $key => $host) { + $this->servers_ []= array('host' => $host, + 'port' => $ports[$key]); + } + } + + /** + * Add a server to the pool + * + * This function does not prevent you from adding a duplicate server entry. + * + * @param string $host hostname or IP + * @param int $port port + */ + public function addServer($host, $port) { + $this->servers_[] = array('host' => $host, 'port' => $port); + } + + /** + * Sets how many time to keep retrying a host in the connect function. + * + * @param int $numRetries + */ + public function setNumRetries($numRetries) { + $this->numRetries_ = $numRetries; + } + + /** + * Sets how long to wait until retrying a host if it was marked down + * + * @param int $numRetries + */ + public function setRetryInterval($retryInterval) { + $this->retryInterval_ = $retryInterval; + } + + /** + * Sets how many time to keep retrying a host before marking it as down. + * + * @param int $numRetries + */ + public function setMaxConsecutiveFailures($maxConsecutiveFailures) { + $this->maxConsecutiveFailures_ = $maxConsecutiveFailures; + } + + /** + * Turns randomization in connect order on or off. + * + * @param bool $randomize + */ + public function setRandomize($randomize) { + $this->randomize_ = $randomize; + } + + /** + * Whether to always try the last server. + * + * @param bool $alwaysTryLast + */ + public function setAlwaysTryLast($alwaysTryLast) { + $this->alwaysTryLast_ = $alwaysTryLast; + } + + + /** + * Connects the socket by iterating through all the servers in the pool + * and trying to find one that works. + */ + public function open() { + // Check if we want order randomization + if ($this->randomize_) { + shuffle($this->servers_); + } + + // Count servers to identify the "last" one + $numServers = count($this->servers_); + + for ($i = 0; $i < $numServers; ++$i) { + + // This extracts the $host and $port variables + extract($this->servers_[$i]); + + // Check APC cache for a record of this server being down + $failtimeKey = 'thrift_failtime:'.$host.':'.$port.'~'; + + // Cache miss? Assume it's OK + $lastFailtime = apc_fetch($failtimeKey); + if ($lastFailtime === FALSE) { + $lastFailtime = 0; + } + + $retryIntervalPassed = FALSE; + + // Cache hit...make sure enough the retry interval has elapsed + if ($lastFailtime > 0) { + $elapsed = time() - $lastFailtime; + if ($elapsed > $this->retryInterval_) { + $retryIntervalPassed = TRUE; + if ($this->debug_) { + call_user_func($this->debugHandler_, + 'TSocketPool: retryInterval '. + '('.$this->retryInterval_.') '. + 'has passed for host '.$host.':'.$port); + } + } + } + + // Only connect if not in the middle of a fail interval, OR if this + // is the LAST server we are trying, just hammer away on it + $isLastServer = FALSE; + if ($this->alwaysTryLast_) { + $isLastServer = ($i == ($numServers - 1)); + } + + if (($lastFailtime === 0) || + ($isLastServer) || + ($lastFailtime > 0 && $retryIntervalPassed)) { + + // Set underlying TSocket params to this one + $this->host_ = $host; + $this->port_ = $port; + + // Try up to numRetries_ connections per server + for ($attempt = 0; $attempt < $this->numRetries_; $attempt++) { + try { + // Use the underlying TSocket open function + parent::open(); + + // Only clear the failure counts if required to do so + if ($lastFailtime > 0) { + apc_store($failtimeKey, 0); + } + + // Successful connection, return now + return; + + } catch (TException $tx) { + // Connection failed + } + } + + // Mark failure of this host in the cache + $consecfailsKey = 'thrift_consecfails:'.$host.':'.$port.'~'; + + // Ignore cache misses + $consecfails = apc_fetch($consecfailsKey); + if ($consecfails === FALSE) { + $consecfails = 0; + } + + // Increment by one + $consecfails++; + + // Log and cache this failure + if ($consecfails >= $this->maxConsecutiveFailures_) { + if ($this->debug_) { + call_user_func($this->debugHandler_, + 'TSocketPool: marking '.$host.':'.$port. + ' as down for '.$this->retryInterval_.' secs '. + 'after '.$consecfails.' failed attempts.'); + } + // Store the failure time + apc_store($failtimeKey, time()); + + // Clear the count of consecutive failures + apc_store($consecfailsKey, 0); + } else { + apc_store($consecfailsKey, $consecfails); + } + } + } + + // Holy shit we failed them all. The system is totally ill! + $error = 'TSocketPool: All hosts in pool are down. '; + $hosts = array(); + foreach ($this->servers_ as $server) { + $hosts []= $server['host'].':'.$server['port']; + } + $hostlist = implode(',', $hosts); + $error .= '('.$hostlist.')'; + if ($this->debug_) { + call_user_func($this->debugHandler_, $error); + } + throw new TException($error); + } +} + +?> diff --git a/lib/php/src/transport/TTransport.php b/lib/php/src/transport/TTransport.php new file mode 100644 index 00000000..e2445259 --- /dev/null +++ b/lib/php/src/transport/TTransport.php @@ -0,0 +1,108 @@ +read($len); + + $data = ''; + $got = 0; + while (($got = strlen($data)) < $len) { + $data .= $this->read($len - $got); + } + return $data; + } + + /** + * Writes the given data out. + * + * @param string $buf The data to write + * @throws TTransportException if writing fails + */ + public abstract function write($buf); + + /** + * Flushes any pending data out of a buffer + * + * @throws TTransportException if a writing error occurs + */ + public function flush() {} +} + +?> diff --git a/lib/py/Makefile.am b/lib/py/Makefile.am new file mode 100644 index 00000000..6d3bbeaa --- /dev/null +++ b/lib/py/Makefile.am @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +DESTDIR ?= / +EXTRA_DIST = setup.py src + +all-local: + $(PYTHON) setup.py build + +# We're ignoring prefix here because site-packages seems to be +# the equivalent of /usr/local/lib in Python land. +# Old version (can't put inline because it's not portable). +#$(PYTHON) setup.py install --prefix=$(prefix) --root=$(DESTDIR) $(PYTHON_SETUPUTIL_ARGS) +install-exec-hook: + $(PYTHON) setup.py install --root=$(DESTDIR) --prefix=$(PY_PREFIX) $(PYTHON_SETUPUTIL_ARGS) + +clean-local: + $(RM) -r build + +check-local: all diff --git a/lib/py/README b/lib/py/README new file mode 100644 index 00000000..29b8c73c --- /dev/null +++ b/lib/py/README @@ -0,0 +1,35 @@ +Thrift Python Software Library + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Using Thrift with Python +======================== + +Thrift is provided as a set of Python packages. The top level package is +thrift, and there are subpackages for the protocol, transport, and server +code. Each package contains modules using standard Thrift naming conventions +(i.e. TProtocol, TTransport) and implementations in corresponding modules +(i.e. TSocket). There is also a subpackage reflection, which contains +the generated code for the reflection structures. + +The Python libraries can be installed manually using the provided setup.py +file, or automatically using the install hook provided via autoconf/automake. +To use the latter, become superuser and do make install. diff --git a/lib/py/setup.py b/lib/py/setup.py new file mode 100644 index 00000000..74837115 --- /dev/null +++ b/lib/py/setup.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from distutils.core import setup, Extension + +fastbinarymod = Extension('thrift.protocol.fastbinary', + sources = ['src/protocol/fastbinary.c'], + ) + +setup(name = 'Thrift', + version = '1.0', + description = 'Thrift Python Libraries', + author = ['Mark Slee'], + author_email = ['mcslee@facebook.com'], + url = 'http://code.facebook.com/thrift', + packages = [ + 'thrift', + 'thrift.protocol', + 'thrift.transport', + 'thrift.server', + ], + package_dir = {'thrift' : 'src'}, + ext_modules = [fastbinarymod], + ) + diff --git a/lib/py/src/TSCons.py b/lib/py/src/TSCons.py new file mode 100644 index 00000000..24046256 --- /dev/null +++ b/lib/py/src/TSCons.py @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from os import path +from SCons.Builder import Builder + +def scons_env(env, add=''): + opath = path.dirname(path.abspath('$TARGET')) + lstr = 'thrift --gen cpp -o ' + opath + ' ' + add + ' $SOURCE' + cppbuild = Builder(action = lstr) + env.Append(BUILDERS = {'ThriftCpp' : cppbuild}) + +def gen_cpp(env, dir, file): + scons_env(env) + suffixes = ['_types.h', '_types.cpp'] + targets = map(lambda s: 'gen-cpp/' + file + s, suffixes) + return env.ThriftCpp(targets, dir+file+'.thrift') diff --git a/lib/py/src/Thrift.py b/lib/py/src/Thrift.py new file mode 100644 index 00000000..21d7aa4e --- /dev/null +++ b/lib/py/src/Thrift.py @@ -0,0 +1,123 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +class TType: + STOP = 0 + VOID = 1 + BOOL = 2 + BYTE = 3 + I08 = 3 + DOUBLE = 4 + I16 = 6 + I32 = 8 + I64 = 10 + STRING = 11 + UTF7 = 11 + STRUCT = 12 + MAP = 13 + SET = 14 + LIST = 15 + UTF8 = 16 + UTF16 = 17 + +class TMessageType: + CALL = 1 + REPLY = 2 + EXCEPTION = 3 + ONEWAY = 4 + +class TProcessor: + + """Base class for procsessor, which works on two streams.""" + + def process(iprot, oprot): + pass + +class TException(Exception): + + """Base class for all thrift exceptions.""" + + def __init__(self, message=None): + Exception.__init__(self, message) + self.message = message + +class TApplicationException(TException): + + """Application level thrift exceptions.""" + + UNKNOWN = 0 + UNKNOWN_METHOD = 1 + INVALID_MESSAGE_TYPE = 2 + WRONG_METHOD_NAME = 3 + BAD_SEQUENCE_ID = 4 + MISSING_RESULT = 5 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + + def __str__(self): + if self.message: + return self.message + elif self.type == UNKNOWN_METHOD: + return 'Unknown method' + elif self.type == INVALID_MESSAGE_TYPE: + return 'Invalid message type' + elif self.type == WRONG_METHOD_NAME: + return 'Wrong method name' + elif self.type == BAD_SEQUENCE_ID: + return 'Bad sequence ID' + elif self.type == MISSING_RESULT: + return 'Missing result' + else: + return 'Default (unknown) TApplicationException' + + def read(self, iprot): + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.message = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.type = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + oprot.writeStructBegin('TApplicationException') + if self.message != None: + oprot.writeFieldBegin('message', TType.STRING, 1) + oprot.writeString(self.message) + oprot.writeFieldEnd() + if self.type != None: + oprot.writeFieldBegin('type', TType.I32, 2) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() diff --git a/lib/py/src/__init__.py b/lib/py/src/__init__.py new file mode 100644 index 00000000..48d659c4 --- /dev/null +++ b/lib/py/src/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['Thrift', 'TSCons'] diff --git a/lib/py/src/protocol/TBinaryProtocol.py b/lib/py/src/protocol/TBinaryProtocol.py new file mode 100644 index 00000000..db1a7a40 --- /dev/null +++ b/lib/py/src/protocol/TBinaryProtocol.py @@ -0,0 +1,259 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TProtocol import * +from struct import pack, unpack + +class TBinaryProtocol(TProtocolBase): + + """Binary implementation of the Thrift protocol driver.""" + + # NastyHaxx. Python 2.4+ on 32-bit machines forces hex constants to be + # positive, converting this into a long. If we hardcode the int value + # instead it'll stay in 32 bit-land. + + # VERSION_MASK = 0xffff0000 + VERSION_MASK = -65536 + + # VERSION_1 = 0x80010000 + VERSION_1 = -2147418112 + + TYPE_MASK = 0x000000ff + + def __init__(self, trans, strictRead=False, strictWrite=True): + TProtocolBase.__init__(self, trans) + self.strictRead = strictRead + self.strictWrite = strictWrite + + def writeMessageBegin(self, name, type, seqid): + if self.strictWrite: + self.writeI32(TBinaryProtocol.VERSION_1 | type) + self.writeString(name) + self.writeI32(seqid) + else: + self.writeString(name) + self.writeByte(type) + self.writeI32(seqid) + + def writeMessageEnd(self): + pass + + def writeStructBegin(self, name): + pass + + def writeStructEnd(self): + pass + + def writeFieldBegin(self, name, type, id): + self.writeByte(type) + self.writeI16(id) + + def writeFieldEnd(self): + pass + + def writeFieldStop(self): + self.writeByte(TType.STOP); + + def writeMapBegin(self, ktype, vtype, size): + self.writeByte(ktype) + self.writeByte(vtype) + self.writeI32(size) + + def writeMapEnd(self): + pass + + def writeListBegin(self, etype, size): + self.writeByte(etype) + self.writeI32(size) + + def writeListEnd(self): + pass + + def writeSetBegin(self, etype, size): + self.writeByte(etype) + self.writeI32(size) + + def writeSetEnd(self): + pass + + def writeBool(self, bool): + if bool: + self.writeByte(1) + else: + self.writeByte(0) + + def writeByte(self, byte): + buff = pack("!b", byte) + self.trans.write(buff) + + def writeI16(self, i16): + buff = pack("!h", i16) + self.trans.write(buff) + + def writeI32(self, i32): + buff = pack("!i", i32) + self.trans.write(buff) + + def writeI64(self, i64): + buff = pack("!q", i64) + self.trans.write(buff) + + def writeDouble(self, dub): + buff = pack("!d", dub) + self.trans.write(buff) + + def writeString(self, str): + self.writeI32(len(str)) + self.trans.write(str) + + def readMessageBegin(self): + sz = self.readI32() + if sz < 0: + version = sz & TBinaryProtocol.VERSION_MASK + if version != TBinaryProtocol.VERSION_1: + raise TProtocolException(TProtocolException.BAD_VERSION, 'Bad version in readMessageBegin: %d' % (sz)) + type = sz & TBinaryProtocol.TYPE_MASK + name = self.readString() + seqid = self.readI32() + else: + if self.strictRead: + raise TProtocolException(TProtocolException.BAD_VERSION, 'No protocol version header') + name = self.trans.readAll(sz) + type = self.readByte() + seqid = self.readI32() + return (name, type, seqid) + + def readMessageEnd(self): + pass + + def readStructBegin(self): + pass + + def readStructEnd(self): + pass + + def readFieldBegin(self): + type = self.readByte() + if type == TType.STOP: + return (None, type, 0) + id = self.readI16() + return (None, type, id) + + def readFieldEnd(self): + pass + + def readMapBegin(self): + ktype = self.readByte() + vtype = self.readByte() + size = self.readI32() + return (ktype, vtype, size) + + def readMapEnd(self): + pass + + def readListBegin(self): + etype = self.readByte() + size = self.readI32() + return (etype, size) + + def readListEnd(self): + pass + + def readSetBegin(self): + etype = self.readByte() + size = self.readI32() + return (etype, size) + + def readSetEnd(self): + pass + + def readBool(self): + byte = self.readByte() + if byte == 0: + return False + return True + + def readByte(self): + buff = self.trans.readAll(1) + val, = unpack('!b', buff) + return val + + def readI16(self): + buff = self.trans.readAll(2) + val, = unpack('!h', buff) + return val + + def readI32(self): + buff = self.trans.readAll(4) + val, = unpack('!i', buff) + return val + + def readI64(self): + buff = self.trans.readAll(8) + val, = unpack('!q', buff) + return val + + def readDouble(self): + buff = self.trans.readAll(8) + val, = unpack('!d', buff) + return val + + def readString(self): + len = self.readI32() + str = self.trans.readAll(len) + return str + + +class TBinaryProtocolFactory: + def __init__(self, strictRead=False, strictWrite=True): + self.strictRead = strictRead + self.strictWrite = strictWrite + + def getProtocol(self, trans): + prot = TBinaryProtocol(trans, self.strictRead, self.strictWrite) + return prot + + +class TBinaryProtocolAccelerated(TBinaryProtocol): + + """C-Accelerated version of TBinaryProtocol. + + This class does not override any of TBinaryProtocol's methods, + but the generated code recognizes it directly and will call into + our C module to do the encoding, bypassing this object entirely. + We inherit from TBinaryProtocol so that the normal TBinaryProtocol + encoding can happen if the fastbinary module doesn't work for some + reason. (TODO(dreiss): Make this happen sanely in more cases.) + + In order to take advantage of the C module, just use + TBinaryProtocolAccelerated instead of TBinaryProtocol. + + NOTE: This code was contributed by an external developer. + The internal Thrift team has reviewed and tested it, + but we cannot guarantee that it is production-ready. + Please feel free to report bugs and/or success stories + to the public mailing list. + """ + + pass + + +class TBinaryProtocolAcceleratedFactory: + def getProtocol(self, trans): + return TBinaryProtocolAccelerated(trans) diff --git a/lib/py/src/protocol/TProtocol.py b/lib/py/src/protocol/TProtocol.py new file mode 100644 index 00000000..be3cb140 --- /dev/null +++ b/lib/py/src/protocol/TProtocol.py @@ -0,0 +1,205 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from thrift.Thrift import * + +class TProtocolException(TException): + + """Custom Protocol Exception class""" + + UNKNOWN = 0 + INVALID_DATA = 1 + NEGATIVE_SIZE = 2 + SIZE_LIMIT = 3 + BAD_VERSION = 4 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + +class TProtocolBase: + + """Base class for Thrift protocol driver.""" + + def __init__(self, trans): + self.trans = trans + + def writeMessageBegin(self, name, type, seqid): + pass + + def writeMessageEnd(self): + pass + + def writeStructBegin(self, name): + pass + + def writeStructEnd(self): + pass + + def writeFieldBegin(self, name, type, id): + pass + + def writeFieldEnd(self): + pass + + def writeFieldStop(self): + pass + + def writeMapBegin(self, ktype, vtype, size): + pass + + def writeMapEnd(self): + pass + + def writeListBegin(self, etype, size): + pass + + def writeListEnd(self): + pass + + def writeSetBegin(self, etype, size): + pass + + def writeSetEnd(self): + pass + + def writeBool(self, bool): + pass + + def writeByte(self, byte): + pass + + def writeI16(self, i16): + pass + + def writeI32(self, i32): + pass + + def writeI64(self, i64): + pass + + def writeDouble(self, dub): + pass + + def writeString(self, str): + pass + + def readMessageBegin(self): + pass + + def readMessageEnd(self): + pass + + def readStructBegin(self): + pass + + def readStructEnd(self): + pass + + def readFieldBegin(self): + pass + + def readFieldEnd(self): + pass + + def readMapBegin(self): + pass + + def readMapEnd(self): + pass + + def readListBegin(self): + pass + + def readListEnd(self): + pass + + def readSetBegin(self): + pass + + def readSetEnd(self): + pass + + def readBool(self): + pass + + def readByte(self): + pass + + def readI16(self): + pass + + def readI32(self): + pass + + def readI64(self): + pass + + def readDouble(self): + pass + + def readString(self): + pass + + def skip(self, type): + if type == TType.STOP: + return + elif type == TType.BOOL: + self.readBool() + elif type == TType.BYTE: + self.readByte() + elif type == TType.I16: + self.readI16() + elif type == TType.I32: + self.readI32() + elif type == TType.I64: + self.readI64() + elif type == TType.DOUBLE: + self.readDouble() + elif type == TType.STRING: + self.readString() + elif type == TType.STRUCT: + name = self.readStructBegin() + while True: + (name, type, id) = self.readFieldBegin() + if type == TType.STOP: + break + self.skip(type) + self.readFieldEnd() + self.readStructEnd() + elif type == TType.MAP: + (ktype, vtype, size) = self.readMapBegin() + for i in range(size): + self.skip(ktype) + self.skip(vtype) + self.readMapEnd() + elif type == TType.SET: + (etype, size) = self.readSetBegin() + for i in range(size): + self.skip(etype) + self.readSetEnd() + elif type == TType.LIST: + (etype, size) = self.readListBegin() + for i in range(size): + self.skip(etype) + self.readListEnd() + +class TProtocolFactory: + def getProtocol(self, trans): + pass diff --git a/lib/py/src/protocol/__init__.py b/lib/py/src/protocol/__init__.py new file mode 100644 index 00000000..01bfe18e --- /dev/null +++ b/lib/py/src/protocol/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TProtocol', 'TBinaryProtocol', 'fastbinary'] diff --git a/lib/py/src/protocol/fastbinary.c b/lib/py/src/protocol/fastbinary.c new file mode 100644 index 00000000..67b215a8 --- /dev/null +++ b/lib/py/src/protocol/fastbinary.c @@ -0,0 +1,1203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include "cStringIO.h" +#include +#include +#include + +/* Fix endianness issues on Solaris */ +#if defined (__SVR4) && defined (__sun) + #if defined(__i386) && !defined(__i386__) + #define __i386__ + #endif + + #ifndef BIG_ENDIAN + #define BIG_ENDIAN (4321) + #endif + #ifndef LITTLE_ENDIAN + #define LITTLE_ENDIAN (1234) + #endif + + /* I386 is LE, even on Solaris */ + #if !defined(BYTE_ORDER) && defined(__i386__) + #define BYTE_ORDER LITTLE_ENDIAN + #endif +#endif + +// TODO(dreiss): defval appears to be unused. Look into removing it. +// TODO(dreiss): Make parse_spec_args recursive, and cache the output +// permanently in the object. (Malloc and orphan.) +// TODO(dreiss): Why do we need cStringIO for reading, why not just char*? +// Can cStringIO let us work with a BufferedTransport? +// TODO(dreiss): Don't ignore the rv from cwrite (maybe). + +/* ====== BEGIN UTILITIES ====== */ + +#define INIT_OUTBUF_SIZE 128 + +// Stolen out of TProtocol.h. +// It would be a huge pain to have both get this from one place. +typedef enum TType { + T_STOP = 0, + T_VOID = 1, + T_BOOL = 2, + T_BYTE = 3, + T_I08 = 3, + T_I16 = 6, + T_I32 = 8, + T_U64 = 9, + T_I64 = 10, + T_DOUBLE = 4, + T_STRING = 11, + T_UTF7 = 11, + T_STRUCT = 12, + T_MAP = 13, + T_SET = 14, + T_LIST = 15, + T_UTF8 = 16, + T_UTF16 = 17 +} TType; + +#ifndef __BYTE_ORDER +# if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN) +# define __BYTE_ORDER BYTE_ORDER +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __BIG_ENDIAN BIG_ENDIAN +# else +# error "Cannot determine endianness" +# endif +#endif + +// Same comment as the enum. Sorry. +#if __BYTE_ORDER == __BIG_ENDIAN +# define ntohll(n) (n) +# define htonll(n) (n) +#elif __BYTE_ORDER == __LITTLE_ENDIAN +# if defined(__GNUC__) && defined(__GLIBC__) +# include +# define ntohll(n) bswap_64(n) +# define htonll(n) bswap_64(n) +# else /* GNUC & GLIBC */ +# define ntohll(n) ( (((unsigned long long)ntohl(n)) << 32) + ntohl(n >> 32) ) +# define htonll(n) ( (((unsigned long long)htonl(n)) << 32) + htonl(n >> 32) ) +# endif /* GNUC & GLIBC */ +#else /* __BYTE_ORDER */ +# error "Can't define htonll or ntohll!" +#endif + +// Doing a benchmark shows that interning actually makes a difference, amazingly. +#define INTERN_STRING(value) _intern_ ## value + +#define INT_CONV_ERROR_OCCURRED(v) ( ((v) == -1) && PyErr_Occurred() ) +#define CHECK_RANGE(v, min, max) ( ((v) <= (max)) && ((v) >= (min)) ) + +// Py_ssize_t was not defined before Python 2.5 +#if (PY_VERSION_HEX < 0x02050000) +typedef int Py_ssize_t; +#endif + +/** + * A cache of the spec_args for a set or list, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + TType element_type; + PyObject* typeargs; +} SetListTypeArgs; + +/** + * A cache of the spec_args for a map, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + TType ktag; + TType vtag; + PyObject* ktypeargs; + PyObject* vtypeargs; +} MapTypeArgs; + +/** + * A cache of the spec_args for a struct, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + PyObject* klass; + PyObject* spec; +} StructTypeArgs; + +/** + * A cache of the item spec from a struct specification, + * so we don't have to keep calling PyTuple_GET_ITEM. + */ +typedef struct { + int tag; + TType type; + PyObject* attrname; + PyObject* typeargs; + PyObject* defval; +} StructItemSpec; + +/** + * A cache of the two key attributes of a CReadableTransport, + * so we don't have to keep calling PyObject_GetAttr. + */ +typedef struct { + PyObject* stringiobuf; + PyObject* refill_callable; +} DecodeBuffer; + +/** Pointer to interned string to speed up attribute lookup. */ +static PyObject* INTERN_STRING(cstringio_buf); +/** Pointer to interned string to speed up attribute lookup. */ +static PyObject* INTERN_STRING(cstringio_refill); + +static inline bool +check_ssize_t_32(Py_ssize_t len) { + // error from getting the int + if (INT_CONV_ERROR_OCCURRED(len)) { + return false; + } + if (!CHECK_RANGE(len, 0, INT32_MAX)) { + PyErr_SetString(PyExc_OverflowError, "string size out of range"); + return false; + } + return true; +} + +static inline bool +parse_pyint(PyObject* o, int32_t* ret, int32_t min, int32_t max) { + long val = PyInt_AsLong(o); + + if (INT_CONV_ERROR_OCCURRED(val)) { + return false; + } + if (!CHECK_RANGE(val, min, max)) { + PyErr_SetString(PyExc_OverflowError, "int out of range"); + return false; + } + + *ret = (int32_t) val; + return true; +} + + +/* --- FUNCTIONS TO PARSE STRUCT SPECIFICATOINS --- */ + +static bool +parse_set_list_args(SetListTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 2) { + PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for list/set type args"); + return false; + } + + dest->element_type = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->element_type)) { + return false; + } + + dest->typeargs = PyTuple_GET_ITEM(typeargs, 1); + + return true; +} + +static bool +parse_map_args(MapTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 4) { + PyErr_SetString(PyExc_TypeError, "expecting 4 arguments for typeargs to map"); + return false; + } + + dest->ktag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->ktag)) { + return false; + } + + dest->vtag = PyInt_AsLong(PyTuple_GET_ITEM(typeargs, 2)); + if (INT_CONV_ERROR_OCCURRED(dest->vtag)) { + return false; + } + + dest->ktypeargs = PyTuple_GET_ITEM(typeargs, 1); + dest->vtypeargs = PyTuple_GET_ITEM(typeargs, 3); + + return true; +} + +static bool +parse_struct_args(StructTypeArgs* dest, PyObject* typeargs) { + if (PyTuple_Size(typeargs) != 2) { + PyErr_SetString(PyExc_TypeError, "expecting tuple of size 2 for struct args"); + return false; + } + + dest->klass = PyTuple_GET_ITEM(typeargs, 0); + dest->spec = PyTuple_GET_ITEM(typeargs, 1); + + return true; +} + +static int +parse_struct_item_spec(StructItemSpec* dest, PyObject* spec_tuple) { + + // i'd like to use ParseArgs here, but it seems to be a bottleneck. + if (PyTuple_Size(spec_tuple) != 5) { + PyErr_SetString(PyExc_TypeError, "expecting 5 arguments for spec tuple"); + return false; + } + + dest->tag = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 0)); + if (INT_CONV_ERROR_OCCURRED(dest->tag)) { + return false; + } + + dest->type = PyInt_AsLong(PyTuple_GET_ITEM(spec_tuple, 1)); + if (INT_CONV_ERROR_OCCURRED(dest->type)) { + return false; + } + + dest->attrname = PyTuple_GET_ITEM(spec_tuple, 2); + dest->typeargs = PyTuple_GET_ITEM(spec_tuple, 3); + dest->defval = PyTuple_GET_ITEM(spec_tuple, 4); + return true; +} + +/* ====== END UTILITIES ====== */ + + +/* ====== BEGIN WRITING FUNCTIONS ====== */ + +/* --- LOW-LEVEL WRITING FUNCTIONS --- */ + +static void writeByte(PyObject* outbuf, int8_t val) { + int8_t net = val; + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int8_t)); +} + +static void writeI16(PyObject* outbuf, int16_t val) { + int16_t net = (int16_t)htons(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int16_t)); +} + +static void writeI32(PyObject* outbuf, int32_t val) { + int32_t net = (int32_t)htonl(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int32_t)); +} + +static void writeI64(PyObject* outbuf, int64_t val) { + int64_t net = (int64_t)htonll(val); + PycStringIO->cwrite(outbuf, (char*)&net, sizeof(int64_t)); +} + +static void writeDouble(PyObject* outbuf, double dub) { + // Unfortunately, bitwise_cast doesn't work in C. Bad C! + union { + double f; + int64_t t; + } transfer; + transfer.f = dub; + writeI64(outbuf, transfer.t); +} + + +/* --- MAIN RECURSIVE OUTPUT FUCNTION -- */ + +static int +output_val(PyObject* output, PyObject* value, TType type, PyObject* typeargs) { + /* + * Refcounting Strategy: + * + * We assume that elements of the thrift_spec tuple are not going to be + * mutated, so we don't ref count those at all. Other than that, we try to + * keep a reference to all the user-created objects while we work with them. + * output_val assumes that a reference is already held. The *caller* is + * responsible for handling references + */ + + switch (type) { + + case T_BOOL: { + int v = PyObject_IsTrue(value); + if (v == -1) { + return false; + } + + writeByte(output, (int8_t) v); + break; + } + case T_I08: { + int32_t val; + + if (!parse_pyint(value, &val, INT8_MIN, INT8_MAX)) { + return false; + } + + writeByte(output, (int8_t) val); + break; + } + case T_I16: { + int32_t val; + + if (!parse_pyint(value, &val, INT16_MIN, INT16_MAX)) { + return false; + } + + writeI16(output, (int16_t) val); + break; + } + case T_I32: { + int32_t val; + + if (!parse_pyint(value, &val, INT32_MIN, INT32_MAX)) { + return false; + } + + writeI32(output, val); + break; + } + case T_I64: { + int64_t nval = PyLong_AsLongLong(value); + + if (INT_CONV_ERROR_OCCURRED(nval)) { + return false; + } + + if (!CHECK_RANGE(nval, INT64_MIN, INT64_MAX)) { + PyErr_SetString(PyExc_OverflowError, "int out of range"); + return false; + } + + writeI64(output, nval); + break; + } + + case T_DOUBLE: { + double nval = PyFloat_AsDouble(value); + if (nval == -1.0 && PyErr_Occurred()) { + return false; + } + + writeDouble(output, nval); + break; + } + + case T_STRING: { + Py_ssize_t len = PyString_Size(value); + + if (!check_ssize_t_32(len)) { + return false; + } + + writeI32(output, (int32_t) len); + PycStringIO->cwrite(output, PyString_AsString(value), (int32_t) len); + break; + } + + case T_LIST: + case T_SET: { + Py_ssize_t len; + SetListTypeArgs parsedargs; + PyObject *item; + PyObject *iterator; + + if (!parse_set_list_args(&parsedargs, typeargs)) { + return false; + } + + len = PyObject_Length(value); + + if (!check_ssize_t_32(len)) { + return false; + } + + writeByte(output, parsedargs.element_type); + writeI32(output, (int32_t) len); + + iterator = PyObject_GetIter(value); + if (iterator == NULL) { + return false; + } + + while ((item = PyIter_Next(iterator))) { + if (!output_val(output, item, parsedargs.element_type, parsedargs.typeargs)) { + Py_DECREF(item); + Py_DECREF(iterator); + return false; + } + Py_DECREF(item); + } + + Py_DECREF(iterator); + + if (PyErr_Occurred()) { + return false; + } + + break; + } + + case T_MAP: { + PyObject *k, *v; + Py_ssize_t pos = 0; + Py_ssize_t len; + + MapTypeArgs parsedargs; + + len = PyDict_Size(value); + if (!check_ssize_t_32(len)) { + return false; + } + + if (!parse_map_args(&parsedargs, typeargs)) { + return false; + } + + writeByte(output, parsedargs.ktag); + writeByte(output, parsedargs.vtag); + writeI32(output, len); + + // TODO(bmaurer): should support any mapping, not just dicts + while (PyDict_Next(value, &pos, &k, &v)) { + // TODO(dreiss): Think hard about whether these INCREFs actually + // turn any unsafe scenarios into safe scenarios. + Py_INCREF(k); + Py_INCREF(v); + + if (!output_val(output, k, parsedargs.ktag, parsedargs.ktypeargs) + || !output_val(output, v, parsedargs.vtag, parsedargs.vtypeargs)) { + Py_DECREF(k); + Py_DECREF(v); + return false; + } + Py_DECREF(k); + Py_DECREF(v); + } + break; + } + + // TODO(dreiss): Consider breaking this out as a function + // the way we did for decode_struct. + case T_STRUCT: { + StructTypeArgs parsedargs; + Py_ssize_t nspec; + Py_ssize_t i; + + if (!parse_struct_args(&parsedargs, typeargs)) { + return false; + } + + nspec = PyTuple_Size(parsedargs.spec); + + if (nspec == -1) { + return false; + } + + for (i = 0; i < nspec; i++) { + StructItemSpec parsedspec; + PyObject* spec_tuple; + PyObject* instval = NULL; + + spec_tuple = PyTuple_GET_ITEM(parsedargs.spec, i); + if (spec_tuple == Py_None) { + continue; + } + + if (!parse_struct_item_spec (&parsedspec, spec_tuple)) { + return false; + } + + instval = PyObject_GetAttr(value, parsedspec.attrname); + + if (!instval) { + return false; + } + + if (instval == Py_None) { + Py_DECREF(instval); + continue; + } + + writeByte(output, (int8_t) parsedspec.type); + writeI16(output, parsedspec.tag); + + if (!output_val(output, instval, parsedspec.type, parsedspec.typeargs)) { + Py_DECREF(instval); + return false; + } + + Py_DECREF(instval); + } + + writeByte(output, (int8_t)T_STOP); + break; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return false; + + } + + return true; +} + + +/* --- TOP-LEVEL WRAPPER FOR OUTPUT -- */ + +static PyObject * +encode_binary(PyObject *self, PyObject *args) { + PyObject* enc_obj; + PyObject* type_args; + PyObject* buf; + PyObject* ret = NULL; + + if (!PyArg_ParseTuple(args, "OO", &enc_obj, &type_args)) { + return NULL; + } + + buf = PycStringIO->NewOutput(INIT_OUTBUF_SIZE); + if (output_val(buf, enc_obj, T_STRUCT, type_args)) { + ret = PycStringIO->cgetvalue(buf); + } + + Py_DECREF(buf); + return ret; +} + +/* ====== END WRITING FUNCTIONS ====== */ + + +/* ====== BEGIN READING FUNCTIONS ====== */ + +/* --- LOW-LEVEL READING FUNCTIONS --- */ + +static void +free_decodebuf(DecodeBuffer* d) { + Py_XDECREF(d->stringiobuf); + Py_XDECREF(d->refill_callable); +} + +static bool +decode_buffer_from_obj(DecodeBuffer* dest, PyObject* obj) { + dest->stringiobuf = PyObject_GetAttr(obj, INTERN_STRING(cstringio_buf)); + if (!dest->stringiobuf) { + return false; + } + + if (!PycStringIO_InputCheck(dest->stringiobuf)) { + free_decodebuf(dest); + PyErr_SetString(PyExc_TypeError, "expecting stringio input"); + return false; + } + + dest->refill_callable = PyObject_GetAttr(obj, INTERN_STRING(cstringio_refill)); + + if(!dest->refill_callable) { + free_decodebuf(dest); + return false; + } + + if (!PyCallable_Check(dest->refill_callable)) { + free_decodebuf(dest); + PyErr_SetString(PyExc_TypeError, "expecting callable"); + return false; + } + + return true; +} + +static bool readBytes(DecodeBuffer* input, char** output, int len) { + int read; + + // TODO(dreiss): Don't fear the malloc. Think about taking a copy of + // the partial read instead of forcing the transport + // to prepend it to its buffer. + + read = PycStringIO->cread(input->stringiobuf, output, len); + + if (read == len) { + return true; + } else if (read == -1) { + return false; + } else { + PyObject* newiobuf; + + // using building functions as this is a rare codepath + newiobuf = PyObject_CallFunction( + input->refill_callable, "s#i", *output, read, len, NULL); + if (newiobuf == NULL) { + return false; + } + + // must do this *AFTER* the call so that we don't deref the io buffer + Py_CLEAR(input->stringiobuf); + input->stringiobuf = newiobuf; + + read = PycStringIO->cread(input->stringiobuf, output, len); + + if (read == len) { + return true; + } else if (read == -1) { + return false; + } else { + // TODO(dreiss): This could be a valid code path for big binary blobs. + PyErr_SetString(PyExc_TypeError, + "refill claimed to have refilled the buffer, but didn't!!"); + return false; + } + } +} + +static int8_t readByte(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int8_t))) { + return -1; + } + + return *(int8_t*) buf; +} + +static int16_t readI16(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int16_t))) { + return -1; + } + + return (int16_t) ntohs(*(int16_t*) buf); +} + +static int32_t readI32(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int32_t))) { + return -1; + } + return (int32_t) ntohl(*(int32_t*) buf); +} + + +static int64_t readI64(DecodeBuffer* input) { + char* buf; + if (!readBytes(input, &buf, sizeof(int64_t))) { + return -1; + } + + return (int64_t) ntohll(*(int64_t*) buf); +} + +static double readDouble(DecodeBuffer* input) { + union { + int64_t f; + double t; + } transfer; + + transfer.f = readI64(input); + if (transfer.f == -1) { + return -1; + } + return transfer.t; +} + +static bool +checkTypeByte(DecodeBuffer* input, TType expected) { + TType got = readByte(input); + if (INT_CONV_ERROR_OCCURRED(got)) { + return false; + } + + if (expected != got) { + PyErr_SetString(PyExc_TypeError, "got wrong ttype while reading field"); + return false; + } + return true; +} + +static bool +skip(DecodeBuffer* input, TType type) { +#define SKIPBYTES(n) \ + do { \ + if (!readBytes(input, &dummy_buf, (n))) { \ + return false; \ + } \ + } while(0) + + char* dummy_buf; + + switch (type) { + + case T_BOOL: + case T_I08: SKIPBYTES(1); break; + case T_I16: SKIPBYTES(2); break; + case T_I32: SKIPBYTES(4); break; + case T_I64: + case T_DOUBLE: SKIPBYTES(8); break; + + case T_STRING: { + // TODO(dreiss): Find out if these check_ssize_t32s are really necessary. + int len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + SKIPBYTES(len); + break; + } + + case T_LIST: + case T_SET: { + TType etype; + int len, i; + + etype = readByte(input); + if (etype == -1) { + return false; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + for (i = 0; i < len; i++) { + if (!skip(input, etype)) { + return false; + } + } + break; + } + + case T_MAP: { + TType ktype, vtype; + int len, i; + + ktype = readByte(input); + if (ktype == -1) { + return false; + } + + vtype = readByte(input); + if (vtype == -1) { + return false; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + for (i = 0; i < len; i++) { + if (!(skip(input, ktype) && skip(input, vtype))) { + return false; + } + } + break; + } + + case T_STRUCT: { + while (true) { + TType type; + + type = readByte(input); + if (type == -1) { + return false; + } + + if (type == T_STOP) + break; + + SKIPBYTES(2); // tag + if (!skip(input, type)) { + return false; + } + } + break; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return false; + + } + + return true; + +#undef SKIPBYTES +} + + +/* --- HELPER FUNCTION FOR DECODE_VAL --- */ + +static PyObject* +decode_val(DecodeBuffer* input, TType type, PyObject* typeargs); + +static bool +decode_struct(DecodeBuffer* input, PyObject* output, PyObject* spec_seq) { + int spec_seq_len = PyTuple_Size(spec_seq); + if (spec_seq_len == -1) { + return false; + } + + while (true) { + TType type; + int16_t tag; + PyObject* item_spec; + PyObject* fieldval = NULL; + StructItemSpec parsedspec; + + type = readByte(input); + if (type == -1) { + return false; + } + if (type == T_STOP) { + break; + } + tag = readI16(input); + if (INT_CONV_ERROR_OCCURRED(tag)) { + return false; + } + if (tag >= 0 && tag < spec_seq_len) { + item_spec = PyTuple_GET_ITEM(spec_seq, tag); + } else { + item_spec = Py_None; + } + + if (item_spec == Py_None) { + if (!skip(input, type)) { + return false; + } else { + continue; + } + } + + if (!parse_struct_item_spec(&parsedspec, item_spec)) { + return false; + } + if (parsedspec.type != type) { + if (!skip(input, type)) { + PyErr_SetString(PyExc_TypeError, "struct field had wrong type while reading and can't be skipped"); + return false; + } else { + continue; + } + } + + fieldval = decode_val(input, parsedspec.type, parsedspec.typeargs); + if (fieldval == NULL) { + return false; + } + + if (PyObject_SetAttr(output, parsedspec.attrname, fieldval) == -1) { + Py_DECREF(fieldval); + return false; + } + Py_DECREF(fieldval); + } + return true; +} + + +/* --- MAIN RECURSIVE INPUT FUCNTION --- */ + +// Returns a new reference. +static PyObject* +decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) { + switch (type) { + + case T_BOOL: { + int8_t v = readByte(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + + switch (v) { + case 0: Py_RETURN_FALSE; + case 1: Py_RETURN_TRUE; + // Don't laugh. This is a potentially serious issue. + default: PyErr_SetString(PyExc_TypeError, "boolean out of range"); return NULL; + } + break; + } + case T_I08: { + int8_t v = readByte(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + + return PyInt_FromLong(v); + } + case T_I16: { + int16_t v = readI16(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + return PyInt_FromLong(v); + } + case T_I32: { + int32_t v = readI32(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + return PyInt_FromLong(v); + } + + case T_I64: { + int64_t v = readI64(input); + if (INT_CONV_ERROR_OCCURRED(v)) { + return NULL; + } + // TODO(dreiss): Find out if we can take this fastpath always when + // sizeof(long) == sizeof(long long). + if (CHECK_RANGE(v, LONG_MIN, LONG_MAX)) { + return PyInt_FromLong((long) v); + } + + return PyLong_FromLongLong(v); + } + + case T_DOUBLE: { + double v = readDouble(input); + if (v == -1.0 && PyErr_Occurred()) { + return false; + } + return PyFloat_FromDouble(v); + } + + case T_STRING: { + Py_ssize_t len = readI32(input); + char* buf; + if (!readBytes(input, &buf, len)) { + return NULL; + } + + return PyString_FromStringAndSize(buf, len); + } + + case T_LIST: + case T_SET: { + SetListTypeArgs parsedargs; + int32_t len; + PyObject* ret = NULL; + int i; + + if (!parse_set_list_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!checkTypeByte(input, parsedargs.element_type)) { + return NULL; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return NULL; + } + + ret = PyList_New(len); + if (!ret) { + return NULL; + } + + for (i = 0; i < len; i++) { + PyObject* item = decode_val(input, parsedargs.element_type, parsedargs.typeargs); + if (!item) { + Py_DECREF(ret); + return NULL; + } + PyList_SET_ITEM(ret, i, item); + } + + // TODO(dreiss): Consider biting the bullet and making two separate cases + // for list and set, avoiding this post facto conversion. + if (type == T_SET) { + PyObject* setret; +#if (PY_VERSION_HEX < 0x02050000) + // hack needed for older versions + setret = PyObject_CallFunctionObjArgs((PyObject*)&PySet_Type, ret, NULL); +#else + // official version + setret = PySet_New(ret); +#endif + Py_DECREF(ret); + return setret; + } + return ret; + } + + case T_MAP: { + int32_t len; + int i; + MapTypeArgs parsedargs; + PyObject* ret = NULL; + + if (!parse_map_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!checkTypeByte(input, parsedargs.ktag)) { + return NULL; + } + if (!checkTypeByte(input, parsedargs.vtag)) { + return NULL; + } + + len = readI32(input); + if (!check_ssize_t_32(len)) { + return false; + } + + ret = PyDict_New(); + if (!ret) { + goto error; + } + + for (i = 0; i < len; i++) { + PyObject* k = NULL; + PyObject* v = NULL; + k = decode_val(input, parsedargs.ktag, parsedargs.ktypeargs); + if (k == NULL) { + goto loop_error; + } + v = decode_val(input, parsedargs.vtag, parsedargs.vtypeargs); + if (v == NULL) { + goto loop_error; + } + if (PyDict_SetItem(ret, k, v) == -1) { + goto loop_error; + } + + Py_DECREF(k); + Py_DECREF(v); + continue; + + // Yuck! Destructors, anyone? + loop_error: + Py_XDECREF(k); + Py_XDECREF(v); + goto error; + } + + return ret; + + error: + Py_XDECREF(ret); + return NULL; + } + + case T_STRUCT: { + StructTypeArgs parsedargs; + if (!parse_struct_args(&parsedargs, typeargs)) { + return NULL; + } + + PyObject* ret = PyObject_CallObject(parsedargs.klass, NULL); + if (!ret) { + return NULL; + } + + if (!decode_struct(input, ret, parsedargs.spec)) { + Py_DECREF(ret); + return NULL; + } + + return ret; + } + + case T_STOP: + case T_VOID: + case T_UTF16: + case T_UTF8: + case T_U64: + default: + PyErr_SetString(PyExc_TypeError, "Unexpected TType"); + return NULL; + } +} + + +/* --- TOP-LEVEL WRAPPER FOR INPUT -- */ + +static PyObject* +decode_binary(PyObject *self, PyObject *args) { + PyObject* output_obj = NULL; + PyObject* transport = NULL; + PyObject* typeargs = NULL; + StructTypeArgs parsedargs; + DecodeBuffer input = {}; + + if (!PyArg_ParseTuple(args, "OOO", &output_obj, &transport, &typeargs)) { + return NULL; + } + + if (!parse_struct_args(&parsedargs, typeargs)) { + return NULL; + } + + if (!decode_buffer_from_obj(&input, transport)) { + return NULL; + } + + if (!decode_struct(&input, output_obj, parsedargs.spec)) { + free_decodebuf(&input); + return NULL; + } + + free_decodebuf(&input); + + Py_RETURN_NONE; +} + +/* ====== END READING FUNCTIONS ====== */ + + +/* -- PYTHON MODULE SETUP STUFF --- */ + +static PyMethodDef ThriftFastBinaryMethods[] = { + + {"encode_binary", encode_binary, METH_VARARGS, ""}, + {"decode_binary", decode_binary, METH_VARARGS, ""}, + + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +PyMODINIT_FUNC +initfastbinary(void) { +#define INIT_INTERN_STRING(value) \ + do { \ + INTERN_STRING(value) = PyString_InternFromString(#value); \ + if(!INTERN_STRING(value)) return; \ + } while(0) + + INIT_INTERN_STRING(cstringio_buf); + INIT_INTERN_STRING(cstringio_refill); +#undef INIT_INTERN_STRING + + PycString_IMPORT; + if (PycStringIO == NULL) return; + + (void) Py_InitModule("thrift.protocol.fastbinary", ThriftFastBinaryMethods); +} diff --git a/lib/py/src/server/THttpServer.py b/lib/py/src/server/THttpServer.py new file mode 100644 index 00000000..21fc3141 --- /dev/null +++ b/lib/py/src/server/THttpServer.py @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import BaseHTTPServer + +from thrift.server import TServer +from thrift.transport import TTransport + +class THttpServer(TServer.TServer): + """A simple HTTP-based Thrift server + + This class is not very performant, but it is useful (for example) for + acting as a mock version of an Apache-based PHP Thrift endpoint.""" + + def __init__(self, processor, server_address, + inputProtocolFactory, outputProtocolFactory = None): + """Set up protocol factories and HTTP server. + + See BaseHTTPServer for server_address. + See TServer for protocol factories.""" + + if outputProtocolFactory is None: + outputProtocolFactory = inputProtocolFactory + + TServer.TServer.__init__(self, processor, None, None, None, + inputProtocolFactory, outputProtocolFactory) + + thttpserver = self + + class RequestHander(BaseHTTPServer.BaseHTTPRequestHandler): + def do_POST(self): + # Don't care about the request path. + self.send_response(200) + self.send_header("content-type", "application/x-thrift") + self.end_headers() + + itrans = TTransport.TFileObjectTransport(self.rfile) + otrans = TTransport.TFileObjectTransport(self.wfile) + iprot = thttpserver.inputProtocolFactory.getProtocol(itrans) + oprot = thttpserver.outputProtocolFactory.getProtocol(otrans) + thttpserver.processor.process(iprot, oprot) + otrans.flush() + + self.httpd = BaseHTTPServer.HTTPServer(server_address, RequestHander) + + def serve(self): + self.httpd.serve_forever() diff --git a/lib/py/src/server/TNonblockingServer.py b/lib/py/src/server/TNonblockingServer.py new file mode 100644 index 00000000..deec708a --- /dev/null +++ b/lib/py/src/server/TNonblockingServer.py @@ -0,0 +1,309 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +"""Implementation of non-blocking server. + +The main idea of the server is reciving and sending requests +only from main thread. + +It also makes thread pool server in tasks terms, not connections. +""" +import threading +import socket +import Queue +import select +import struct +import logging + +from thrift.transport import TTransport +from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory + +__all__ = ['TNonblockingServer'] + +class Worker(threading.Thread): + """Worker is a small helper to process incoming connection.""" + def __init__(self, queue): + threading.Thread.__init__(self) + self.queue = queue + + def run(self): + """Process queries from task queue, stop if processor is None.""" + while True: + try: + processor, iprot, oprot, otrans, callback = self.queue.get() + if processor is None: + break + processor.process(iprot, oprot) + callback(True, otrans.getvalue()) + except Exception: + logging.exception("Exception while processing request") + callback(False, '') + +WAIT_LEN = 0 +WAIT_MESSAGE = 1 +WAIT_PROCESS = 2 +SEND_ANSWER = 3 +CLOSED = 4 + +def locked(func): + "Decorator which locks self.lock." + def nested(self, *args, **kwargs): + self.lock.acquire() + try: + return func(self, *args, **kwargs) + finally: + self.lock.release() + return nested + +def socket_exception(func): + "Decorator close object on socket.error." + def read(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except socket.error: + self.close() + return read + +class Connection: + """Basic class is represented connection. + + It can be in state: + WAIT_LEN --- connection is reading request len. + WAIT_MESSAGE --- connection is reading request. + WAIT_PROCESS --- connection has just read whole request and + waits for call ready routine. + SEND_ANSWER --- connection is sending answer string (including length + of answer). + CLOSED --- socket was closed and connection should be deleted. + """ + def __init__(self, new_socket, wake_up): + self.socket = new_socket + self.socket.setblocking(False) + self.status = WAIT_LEN + self.len = 0 + self.message = '' + self.lock = threading.Lock() + self.wake_up = wake_up + + def _read_len(self): + """Reads length of request. + + It's really paranoic routine and it may be replaced by + self.socket.recv(4).""" + read = self.socket.recv(4 - len(self.message)) + if len(read) == 0: + # if we read 0 bytes and self.message is empty, it means client close + # connection + if len(self.message) != 0: + logging.error("can't read frame size from socket") + self.close() + return + self.message += read + if len(self.message) == 4: + self.len, = struct.unpack('!i', self.message) + if self.len < 0: + logging.error("negative frame size, it seems client"\ + " doesn't use FramedTransport") + self.close() + elif self.len == 0: + logging.error("empty frame, it's really strange") + self.close() + else: + self.message = '' + self.status = WAIT_MESSAGE + + @socket_exception + def read(self): + """Reads data from stream and switch state.""" + assert self.status in (WAIT_LEN, WAIT_MESSAGE) + if self.status == WAIT_LEN: + self._read_len() + # go back to the main loop here for simplicity instead of + # falling through, even though there is a good chance that + # the message is already available + elif self.status == WAIT_MESSAGE: + read = self.socket.recv(self.len - len(self.message)) + if len(read) == 0: + logging.error("can't read frame from socket (get %d of %d bytes)" % + (len(self.message), self.len)) + self.close() + return + self.message += read + if len(self.message) == self.len: + self.status = WAIT_PROCESS + + @socket_exception + def write(self): + """Writes data from socket and switch state.""" + assert self.status == SEND_ANSWER + sent = self.socket.send(self.message) + if sent == len(self.message): + self.status = WAIT_LEN + self.message = '' + self.len = 0 + else: + self.message = self.message[sent:] + + @locked + def ready(self, all_ok, message): + """Callback function for switching state and waking up main thread. + + This function is the only function witch can be called asynchronous. + + The ready can switch Connection to three states: + WAIT_LEN if request was oneway. + SEND_ANSWER if request was processed in normal way. + CLOSED if request throws unexpected exception. + + The one wakes up main thread. + """ + assert self.status == WAIT_PROCESS + if not all_ok: + self.close() + self.wake_up() + return + self.len = '' + self.message = struct.pack('!i', len(message)) + message + if len(message) == 0: + # it was a oneway request, do not write answer + self.status = WAIT_LEN + else: + self.status = SEND_ANSWER + self.wake_up() + + @locked + def is_writeable(self): + "Returns True if connection should be added to write list of select." + return self.status == SEND_ANSWER + + # it's not necessary, but... + @locked + def is_readable(self): + "Returns True if connection should be added to read list of select." + return self.status in (WAIT_LEN, WAIT_MESSAGE) + + @locked + def is_closed(self): + "Returns True if connection is closed." + return self.status == CLOSED + + def fileno(self): + "Returns the file descriptor of the associated socket." + return self.socket.fileno() + + def close(self): + "Closes connection" + self.status = CLOSED + self.socket.close() + +class TNonblockingServer: + """Non-blocking server.""" + def __init__(self, processor, lsocket, inputProtocolFactory=None, + outputProtocolFactory=None, threads=10): + self.processor = processor + self.socket = lsocket + self.in_protocol = inputProtocolFactory or TBinaryProtocolFactory() + self.out_protocol = outputProtocolFactory or self.in_protocol + self.threads = int(threads) + self.clients = {} + self.tasks = Queue.Queue() + self._read, self._write = socket.socketpair() + self.prepared = False + + def setNumThreads(self, num): + """Set the number of worker threads that should be created.""" + # implement ThreadPool interface + assert not self.prepared, "You can't change number of threads for working server" + self.threads = num + + def prepare(self): + """Prepares server for serve requests.""" + self.socket.listen() + for _ in xrange(self.threads): + thread = Worker(self.tasks) + thread.setDaemon(True) + thread.start() + self.prepared = True + + def wake_up(self): + """Wake up main thread. + + The server usualy waits in select call in we should terminate one. + The simplest way is using socketpair. + + Select always wait to read from the first socket of socketpair. + + In this case, we can just write anything to the second socket from + socketpair.""" + self._write.send('1') + + def _select(self): + """Does select on open connections.""" + readable = [self.socket.handle.fileno(), self._read.fileno()] + writable = [] + for i, connection in self.clients.items(): + if connection.is_readable(): + readable.append(connection.fileno()) + if connection.is_writeable(): + writable.append(connection.fileno()) + if connection.is_closed(): + del self.clients[i] + return select.select(readable, writable, readable) + + def handle(self): + """Handle requests. + + WARNING! You must call prepare BEFORE calling handle. + """ + assert self.prepared, "You have to call prepare before handle" + rset, wset, xset = self._select() + for readable in rset: + if readable == self._read.fileno(): + # don't care i just need to clean readable flag + self._read.recv(1024) + elif readable == self.socket.handle.fileno(): + client = self.socket.accept().handle + self.clients[client.fileno()] = Connection(client, self.wake_up) + else: + connection = self.clients[readable] + connection.read() + if connection.status == WAIT_PROCESS: + itransport = TTransport.TMemoryBuffer(connection.message) + otransport = TTransport.TMemoryBuffer() + iprot = self.in_protocol.getProtocol(itransport) + oprot = self.out_protocol.getProtocol(otransport) + self.tasks.put([self.processor, iprot, oprot, + otransport, connection.ready]) + for writeable in wset: + self.clients[writeable].write() + for oob in xset: + self.clients[oob].close() + del self.clients[oob] + + def close(self): + """Closes the server.""" + for _ in xrange(self.threads): + self.tasks.put([None, None, None, None, None]) + self.socket.close() + self.prepared = False + + def serve(self): + """Serve forever.""" + self.prepare() + while True: + self.handle() diff --git a/lib/py/src/server/TServer.py b/lib/py/src/server/TServer.py new file mode 100644 index 00000000..61529111 --- /dev/null +++ b/lib/py/src/server/TServer.py @@ -0,0 +1,270 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import logging +import sys +import os +import traceback +import threading +import Queue + +from thrift.Thrift import TProcessor +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol + +class TServer: + + """Base interface for a server, which must have a serve method.""" + + """ 3 constructors for all servers: + 1) (processor, serverTransport) + 2) (processor, serverTransport, transportFactory, protocolFactory) + 3) (processor, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory)""" + def __init__(self, *args): + if (len(args) == 2): + self.__initArgs__(args[0], args[1], + TTransport.TTransportFactoryBase(), + TTransport.TTransportFactoryBase(), + TBinaryProtocol.TBinaryProtocolFactory(), + TBinaryProtocol.TBinaryProtocolFactory()) + elif (len(args) == 4): + self.__initArgs__(args[0], args[1], args[2], args[2], args[3], args[3]) + elif (len(args) == 6): + self.__initArgs__(args[0], args[1], args[2], args[3], args[4], args[5]) + + def __initArgs__(self, processor, serverTransport, + inputTransportFactory, outputTransportFactory, + inputProtocolFactory, outputProtocolFactory): + self.processor = processor + self.serverTransport = serverTransport + self.inputTransportFactory = inputTransportFactory + self.outputTransportFactory = outputTransportFactory + self.inputProtocolFactory = inputProtocolFactory + self.outputProtocolFactory = outputProtocolFactory + + def serve(self): + pass + +class TSimpleServer(TServer): + + """Simple single-threaded server that just pumps around one transport.""" + + def __init__(self, *args): + TServer.__init__(self, *args) + + def serve(self): + self.serverTransport.listen() + while True: + client = self.serverTransport.accept() + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + +class TThreadedServer(TServer): + + """Threaded server that spawns a new thread per each connection.""" + + def __init__(self, *args): + TServer.__init__(self, *args) + + def serve(self): + self.serverTransport.listen() + while True: + try: + client = self.serverTransport.accept() + t = threading.Thread(target = self.handle, args=(client,)) + t.start() + except KeyboardInterrupt: + raise + except Exception, x: + logging.exception(x) + + def handle(self, client): + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + +class TThreadPoolServer(TServer): + + """Server with a fixed size pool of threads which service requests.""" + + def __init__(self, *args): + TServer.__init__(self, *args) + self.clients = Queue.Queue() + self.threads = 10 + + def setNumThreads(self, num): + """Set the number of worker threads that should be created""" + self.threads = num + + def serveThread(self): + """Loop around getting clients from the shared queue and process them.""" + while True: + try: + client = self.clients.get() + self.serveClient(client) + except Exception, x: + logging.exception(x) + + def serveClient(self, client): + """Process input/output from a client for as long as possible""" + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + itrans.close() + otrans.close() + + def serve(self): + """Start a fixed number of worker threads and put client into a queue""" + for i in range(self.threads): + try: + t = threading.Thread(target = self.serveThread) + t.start() + except Exception, x: + logging.exception(x) + + # Pump the socket for clients + self.serverTransport.listen() + while True: + try: + client = self.serverTransport.accept() + self.clients.put(client) + except Exception, x: + logging.exception(x) + + +class TForkingServer(TServer): + + """A Thrift server that forks a new process for each request""" + """ + This is more scalable than the threaded server as it does not cause + GIL contention. + + Note that this has different semantics from the threading server. + Specifically, updates to shared variables will no longer be shared. + It will also not work on windows. + + This code is heavily inspired by SocketServer.ForkingMixIn in the + Python stdlib. + """ + + def __init__(self, *args): + TServer.__init__(self, *args) + self.children = [] + + def serve(self): + def try_close(file): + try: + file.close() + except IOError, e: + logging.warning(e, exc_info=True) + + + self.serverTransport.listen() + while True: + client = self.serverTransport.accept() + try: + pid = os.fork() + + if pid: # parent + # add before collect, otherwise you race w/ waitpid + self.children.append(pid) + self.collect_children() + + # Parent must close socket or the connection may not get + # closed promptly + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + try_close(itrans) + try_close(otrans) + else: + itrans = self.inputTransportFactory.getTransport(client) + otrans = self.outputTransportFactory.getTransport(client) + + iprot = self.inputProtocolFactory.getProtocol(itrans) + oprot = self.outputProtocolFactory.getProtocol(otrans) + + ecode = 0 + try: + try: + while True: + self.processor.process(iprot, oprot) + except TTransport.TTransportException, tx: + pass + except Exception, e: + logging.exception(e) + ecode = 1 + finally: + try_close(itrans) + try_close(otrans) + + os._exit(ecode) + + except TTransport.TTransportException, tx: + pass + except Exception, x: + logging.exception(x) + + + def collect_children(self): + while self.children: + try: + pid, status = os.waitpid(0, os.WNOHANG) + except os.error: + pid = None + + if pid: + self.children.remove(pid) + else: + break + + diff --git a/lib/py/src/server/__init__.py b/lib/py/src/server/__init__.py new file mode 100644 index 00000000..1bf6e254 --- /dev/null +++ b/lib/py/src/server/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TServer', 'TNonblockingServer'] diff --git a/lib/py/src/transport/THttpClient.py b/lib/py/src/transport/THttpClient.py new file mode 100644 index 00000000..5086032b --- /dev/null +++ b/lib/py/src/transport/THttpClient.py @@ -0,0 +1,100 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TTransport import * +from cStringIO import StringIO + +import urlparse +import httplib +import warnings + +class THttpClient(TTransportBase): + + """Http implementation of TTransport base.""" + + def __init__(self, uri_or_host, port=None, path=None): + """THttpClient supports two different types constructor parameters. + + THttpClient(host, port, path) - deprecated + THttpClient(uri) + + Only the second supports https.""" + + if port is not None: + warnings.warn("Please use the THttpClient('http://host:port/path') syntax", DeprecationWarning, stacklevel=2) + self.host = uri_or_host + self.port = port + assert path + self.path = path + self.scheme = 'http' + else: + parsed = urlparse.urlparse(uri_or_host) + self.scheme = parsed.scheme + assert self.scheme in ('http', 'https') + if self.scheme == 'http': + self.port = parsed.port or httplib.HTTP_PORT + elif self.scheme == 'https': + self.port = parsed.port or httplib.HTTPS_PORT + self.host = parsed.hostname + self.path = parsed.path + self.__wbuf = StringIO() + self.__http = None + + def open(self): + if self.scheme == 'http': + self.__http = httplib.HTTP(self.host, self.port) + else: + self.__http = httplib.HTTPS(self.host, self.port) + + def close(self): + self.__http.close() + self.__http = None + + def isOpen(self): + return self.__http != None + + def read(self, sz): + return self.__http.file.read(sz) + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + if self.isOpen(): + self.close() + self.open(); + + # Pull data out of buffer + data = self.__wbuf.getvalue() + self.__wbuf = StringIO() + + # HTTP request + self.__http.putrequest('POST', self.path) + + # Write headers + self.__http.putheader('Host', self.host) + self.__http.putheader('Content-Type', 'application/x-thrift') + self.__http.putheader('Content-Length', str(len(data))) + self.__http.endheaders() + + # Write payload + self.__http.send(data) + + # Get reply to flush the request + self.code, self.message, self.headers = self.__http.getreply() diff --git a/lib/py/src/transport/TSocket.py b/lib/py/src/transport/TSocket.py new file mode 100644 index 00000000..4645a023 --- /dev/null +++ b/lib/py/src/transport/TSocket.py @@ -0,0 +1,147 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from TTransport import * +import os +import errno +import socket + +class TSocketBase(TTransportBase): + def _resolveAddr(self): + if self._unix_socket is not None: + return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None, self._unix_socket)] + else: + return socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE | socket.AI_ADDRCONFIG) + + def close(self): + if self.handle: + self.handle.close() + self.handle = None + +class TSocket(TSocketBase): + """Socket implementation of TTransport base.""" + + def __init__(self, host='localhost', port=9090, unix_socket=None): + """Initialize a TSocket + + @param host(str) The host to connect to. + @param port(int) The (TCP) port to connect to. + @param unix_socket(str) The filename of a unix socket to connect to. + (host and port will be ignored.) + """ + + self.host = host + self.port = port + self.handle = None + self._unix_socket = unix_socket + self._timeout = None + + def setHandle(self, h): + self.handle = h + + def isOpen(self): + return self.handle != None + + def setTimeout(self, ms): + if ms is None: + self._timeout = None + else: + self._timeout = ms/1000.0 + + if (self.handle != None): + self.handle.settimeout(self._timeout) + + def open(self): + try: + res0 = self._resolveAddr() + for res in res0: + self.handle = socket.socket(res[0], res[1]) + self.handle.settimeout(self._timeout) + try: + self.handle.connect(res[4]) + except socket.error, e: + if res is not res0[-1]: + continue + else: + raise e + break + except socket.error, e: + if self._unix_socket: + message = 'Could not connect to socket %s' % self._unix_socket + else: + message = 'Could not connect to %s:%d' % (self.host, self.port) + raise TTransportException(TTransportException.NOT_OPEN, message) + + def read(self, sz): + buff = self.handle.recv(sz) + if len(buff) == 0: + raise TTransportException('TSocket read 0 bytes') + return buff + + def write(self, buff): + sent = 0 + have = len(buff) + while sent < have: + plus = self.handle.send(buff) + if plus == 0: + raise TTransportException('TSocket sent 0 bytes') + sent += plus + buff = buff[plus:] + + def flush(self): + pass + +class TServerSocket(TSocketBase, TServerTransportBase): + """Socket implementation of TServerTransport base.""" + + def __init__(self, port=9090, unix_socket=None): + self.host = None + self.port = port + self._unix_socket = unix_socket + self.handle = None + + def listen(self): + res0 = self._resolveAddr() + for res in res0: + if res[0] is socket.AF_INET6 or res is res0[-1]: + break + + # We need remove the old unix socket if the file exists and + # nobody is listening on it. + if self._unix_socket: + tmp = socket.socket(res[0], res[1]) + try: + tmp.connect(res[4]) + except socket.error, err: + eno, message = err.args + if eno == errno.ECONNREFUSED: + os.unlink(res[4]) + + self.handle = socket.socket(res[0], res[1]) + self.handle.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(self.handle, 'set_timeout'): + self.handle.set_timeout(None) + self.handle.bind(res[4]) + self.handle.listen(128) + + def accept(self): + client, addr = self.handle.accept() + result = TSocket() + result.setHandle(client) + return result diff --git a/lib/py/src/transport/TTransport.py b/lib/py/src/transport/TTransport.py new file mode 100644 index 00000000..32553bdd --- /dev/null +++ b/lib/py/src/transport/TTransport.py @@ -0,0 +1,326 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from cStringIO import StringIO +from struct import pack,unpack +from thrift.Thrift import TException + +class TTransportException(TException): + + """Custom Transport Exception class""" + + UNKNOWN = 0 + NOT_OPEN = 1 + ALREADY_OPEN = 2 + TIMED_OUT = 3 + END_OF_FILE = 4 + + def __init__(self, type=UNKNOWN, message=None): + TException.__init__(self, message) + self.type = type + +class TTransportBase: + + """Base class for Thrift transport layer.""" + + def isOpen(self): + pass + + def open(self): + pass + + def close(self): + pass + + def read(self, sz): + pass + + def readAll(self, sz): + buff = '' + have = 0 + while (have < sz): + chunk = self.read(sz-have) + have += len(chunk) + buff += chunk + + if len(chunk) == 0: + raise EOFError() + + return buff + + def write(self, buf): + pass + + def flush(self): + pass + +# This class should be thought of as an interface. +class CReadableTransport: + """base class for transports that are readable from C""" + + # TODO(dreiss): Think about changing this interface to allow us to use + # a (Python, not c) StringIO instead, because it allows + # you to write after reading. + + # NOTE: This is a classic class, so properties will NOT work + # correctly for setting. + @property + def cstringio_buf(self): + """A cStringIO buffer that contains the current chunk we are reading.""" + pass + + def cstringio_refill(self, partialread, reqlen): + """Refills cstringio_buf. + + Returns the currently used buffer (which can but need not be the same as + the old cstringio_buf). partialread is what the C code has read from the + buffer, and should be inserted into the buffer before any more reads. The + return value must be a new, not borrowed reference. Something along the + lines of self._buf should be fine. + + If reqlen bytes can't be read, throw EOFError. + """ + pass + +class TServerTransportBase: + + """Base class for Thrift server transports.""" + + def listen(self): + pass + + def accept(self): + pass + + def close(self): + pass + +class TTransportFactoryBase: + + """Base class for a Transport Factory""" + + def getTransport(self, trans): + return trans + +class TBufferedTransportFactory: + + """Factory transport that builds buffered transports""" + + def getTransport(self, trans): + buffered = TBufferedTransport(trans) + return buffered + + +class TBufferedTransport(TTransportBase,CReadableTransport): + + """Class that wraps another transport and buffers its I/O.""" + + DEFAULT_BUFFER = 4096 + + def __init__(self, trans): + self.__trans = trans + self.__wbuf = StringIO() + self.__rbuf = StringIO("") + + def isOpen(self): + return self.__trans.isOpen() + + def open(self): + return self.__trans.open() + + def close(self): + return self.__trans.close() + + def read(self, sz): + ret = self.__rbuf.read(sz) + if len(ret) != 0: + return ret + + self.__rbuf = StringIO(self.__trans.read(max(sz, self.DEFAULT_BUFFER))) + return self.__rbuf.read(sz) + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + out = self.__wbuf.getvalue() + # reset wbuf before write/flush to preserve state on underlying failure + self.__wbuf = StringIO() + self.__trans.write(out) + self.__trans.flush() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self.__rbuf + + def cstringio_refill(self, partialread, reqlen): + retstring = partialread + if reqlen < self.DEFAULT_BUFFER: + # try to make a read of as much as we can. + retstring += self.__trans.read(self.DEFAULT_BUFFER) + + # but make sure we do read reqlen bytes. + if len(retstring) < reqlen: + retstring += self.__trans.readAll(reqlen - len(retstring)) + + self.__rbuf = StringIO(retstring) + return self.__rbuf + +class TMemoryBuffer(TTransportBase, CReadableTransport): + """Wraps a cStringIO object as a TTransport. + + NOTE: Unlike the C++ version of this class, you cannot write to it + then immediately read from it. If you want to read from a + TMemoryBuffer, you must either pass a string to the constructor. + TODO(dreiss): Make this work like the C++ version. + """ + + def __init__(self, value=None): + """value -- a value to read from for stringio + + If value is set, this will be a transport for reading, + otherwise, it is for writing""" + if value is not None: + self._buffer = StringIO(value) + else: + self._buffer = StringIO() + + def isOpen(self): + return not self._buffer.closed + + def open(self): + pass + + def close(self): + self._buffer.close() + + def read(self, sz): + return self._buffer.read(sz) + + def write(self, buf): + self._buffer.write(buf) + + def flush(self): + pass + + def getvalue(self): + return self._buffer.getvalue() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self._buffer + + def cstringio_refill(self, partialread, reqlen): + # only one shot at reading... + raise EOFError() + +class TFramedTransportFactory: + + """Factory transport that builds framed transports""" + + def getTransport(self, trans): + framed = TFramedTransport(trans) + return framed + + +class TFramedTransport(TTransportBase, CReadableTransport): + + """Class that wraps another transport and frames its I/O when writing.""" + + def __init__(self, trans,): + self.__trans = trans + self.__rbuf = StringIO() + self.__wbuf = StringIO() + + def isOpen(self): + return self.__trans.isOpen() + + def open(self): + return self.__trans.open() + + def close(self): + return self.__trans.close() + + def read(self, sz): + ret = self.__rbuf.read(sz) + if len(ret) != 0: + return ret + + self.readFrame() + return self.__rbuf.read(sz) + + def readFrame(self): + buff = self.__trans.readAll(4) + sz, = unpack('!i', buff) + self.__rbuf = StringIO(self.__trans.readAll(sz)) + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + wout = self.__wbuf.getvalue() + wsz = len(wout) + # reset wbuf before write/flush to preserve state on underlying failure + self.__wbuf = StringIO() + # N.B.: Doing this string concatenation is WAY cheaper than making + # two separate calls to the underlying socket object. Socket writes in + # Python turn out to be REALLY expensive, but it seems to do a pretty + # good job of managing string buffer operations without excessive copies + buf = pack("!i", wsz) + wout + self.__trans.write(buf) + self.__trans.flush() + + # Implement the CReadableTransport interface. + @property + def cstringio_buf(self): + return self.__rbuf + + def cstringio_refill(self, prefix, reqlen): + # self.__rbuf will already be empty here because fastbinary doesn't + # ask for a refill until the previous buffer is empty. Therefore, + # we can start reading new frames immediately. + while len(prefix) < reqlen: + readFrame() + prefix += self.__rbuf.getvalue() + self.__rbuf = StringIO(prefix) + return self.__rbuf + + +class TFileObjectTransport(TTransportBase): + """Wraps a file-like object to make it work as a Thrift transport.""" + + def __init__(self, fileobj): + self.fileobj = fileobj + + def isOpen(self): + return True + + def close(self): + self.fileobj.close() + + def read(self, sz): + return self.fileobj.read(sz) + + def write(self, buf): + self.fileobj.write(buf) + + def flush(self): + self.fileobj.flush() diff --git a/lib/py/src/transport/TTwisted.py b/lib/py/src/transport/TTwisted.py new file mode 100644 index 00000000..b5c2147b --- /dev/null +++ b/lib/py/src/transport/TTwisted.py @@ -0,0 +1,177 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from zope.interface import implements, Interface, Attribute +from twisted.internet.protocol import Protocol, ServerFactory, ClientFactory, \ + connectionDone +from twisted.internet import defer +from twisted.protocols import basic +from twisted.python import log + + +from thrift.transport import TTransport +from cStringIO import StringIO + + +class TMessageSenderTransport(TTransport.TTransportBase): + + def __init__(self): + self.__wbuf = StringIO() + + def write(self, buf): + self.__wbuf.write(buf) + + def flush(self): + msg = self.__wbuf.getvalue() + self.__wbuf = StringIO() + self.sendMessage(msg) + + def sendMessage(self, message): + raise NotImplementedError + + +class TCallbackTransport(TMessageSenderTransport): + + def __init__(self, func): + TMessageSenderTransport.__init__(self) + self.func = func + + def sendMessage(self, message): + self.func(message) + + +class ThriftClientProtocol(basic.Int32StringReceiver): + + def __init__(self, client_class, iprot_factory, oprot_factory=None): + self._client_class = client_class + self._iprot_factory = iprot_factory + if oprot_factory is None: + self._oprot_factory = iprot_factory + else: + self._oprot_factory = oprot_factory + + self.recv_map = {} + self.started = defer.Deferred() + + def dispatch(self, msg): + self.sendString(msg) + + def connectionMade(self): + tmo = TCallbackTransport(self.dispatch) + self.client = self._client_class(tmo, self._oprot_factory) + self.started.callback(self.client) + + def connectionLost(self, reason=connectionDone): + for k,v in self.client._reqs.iteritems(): + tex = TTransport.TTransportException( + type=TTransport.TTransportException.END_OF_FILE, + message='Connection closed') + v.errback(tex) + + def stringReceived(self, frame): + tr = TTransport.TMemoryBuffer(frame) + iprot = self._iprot_factory.getProtocol(tr) + (fname, mtype, rseqid) = iprot.readMessageBegin() + + try: + method = self.recv_map[fname] + except KeyError: + method = getattr(self.client, 'recv_' + fname) + self.recv_map[fname] = method + + method(iprot, mtype, rseqid) + + +class ThriftServerProtocol(basic.Int32StringReceiver): + + def dispatch(self, msg): + self.sendString(msg) + + def processError(self, error): + self.transport.loseConnection() + + def processOk(self, _, tmo): + msg = tmo.getvalue() + + if len(msg) > 0: + self.dispatch(msg) + + def stringReceived(self, frame): + tmi = TTransport.TMemoryBuffer(frame) + tmo = TTransport.TMemoryBuffer() + + iprot = self.factory.iprot_factory.getProtocol(tmi) + oprot = self.factory.oprot_factory.getProtocol(tmo) + + d = self.factory.processor.process(iprot, oprot) + d.addCallbacks(self.processOk, self.processError, + callbackArgs=(tmo,)) + + +class IThriftServerFactory(Interface): + + processor = Attribute("Thrift processor") + + iprot_factory = Attribute("Input protocol factory") + + oprot_factory = Attribute("Output protocol factory") + + +class IThriftClientFactory(Interface): + + client_class = Attribute("Thrift client class") + + iprot_factory = Attribute("Input protocol factory") + + oprot_factory = Attribute("Output protocol factory") + + +class ThriftServerFactory(ServerFactory): + + implements(IThriftServerFactory) + + protocol = ThriftServerProtocol + + def __init__(self, processor, iprot_factory, oprot_factory=None): + self.processor = processor + self.iprot_factory = iprot_factory + if oprot_factory is None: + self.oprot_factory = iprot_factory + else: + self.oprot_factory = oprot_factory + + +class ThriftClientFactory(ClientFactory): + + implements(IThriftClientFactory) + + protocol = ThriftClientProtocol + + def __init__(self, client_class, iprot_factory, oprot_factory=None): + self.client_class = client_class + self.iprot_factory = iprot_factory + if oprot_factory is None: + self.oprot_factory = iprot_factory + else: + self.oprot_factory = oprot_factory + + def buildProtocol(self, addr): + p = self.protocol(self.client_class, self.iprot_factory, + self.oprot_factory) + p.factory = self + return p diff --git a/lib/py/src/transport/__init__.py b/lib/py/src/transport/__init__.py new file mode 100644 index 00000000..02c6048a --- /dev/null +++ b/lib/py/src/transport/__init__.py @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +__all__ = ['TTransport', 'TSocket', 'THttpClient'] diff --git a/lib/rb/CHANGELOG b/lib/rb/CHANGELOG new file mode 100644 index 00000000..b5dce2ae --- /dev/null +++ b/lib/rb/CHANGELOG @@ -0,0 +1 @@ +v0.0.1. Initial release diff --git a/lib/rb/Makefile.am b/lib/rb/Makefile.am new file mode 100644 index 00000000..9cfffc71 --- /dev/null +++ b/lib/rb/Makefile.am @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +EXTRA_DIST = \ + CHANGELOG \ + Rakefile \ + Manifest \ + setup.rb \ + lib \ + ext \ + benchmark \ + script \ + spec + +all-local: + $(RUBY) setup.rb config + $(RUBY) setup.rb setup + +install-exec-hook: + $(RUBY) setup.rb install + +# Make sure this doesn't fail if Ruby is not configured. +clean-local: + RUBY=$(RUBY) ; if test -z "$$RUBY" ; then RUBY=: ; fi ; \ + $$RUBY setup.rb clean + +check-local: all +if HAVE_RSPEC + rake spec +endif + diff --git a/lib/rb/Manifest b/lib/rb/Manifest new file mode 100644 index 00000000..7b4503fb --- /dev/null +++ b/lib/rb/Manifest @@ -0,0 +1,81 @@ +CHANGELOG +Manifest +Rakefile +README +setup.rb +benchmark/benchmark.rb +benchmark/Benchmark.thrift +benchmark/client.rb +benchmark/server.rb +benchmark/thin_server.rb +ext/binary_protocol_accelerated.c +ext/binary_protocol_accelerated.h +ext/compact_protocol.c +ext/compact_protocol.h +ext/constants.h +ext/extconf.rb +ext/macros.h +ext/memory_buffer.c +ext/memory_buffer.h +ext/protocol.c +ext/protocol.h +ext/struct.c +ext/struct.h +ext/thrift_native.c +lib/thrift.rb +lib/thrift/client.rb +lib/thrift/core_ext.rb +lib/thrift/exceptions.rb +lib/thrift/processor.rb +lib/thrift/struct.rb +lib/thrift/thrift_native.rb +lib/thrift/types.rb +lib/thrift/core_ext/fixnum.rb +lib/thrift/protocol/base_protocol.rb +lib/thrift/protocol/binary_protocol.rb +lib/thrift/protocol/binary_protocol_accelerated.rb +lib/thrift/protocol/compact_protocol.rb +lib/thrift/serializer/deserializer.rb +lib/thrift/serializer/serializer.rb +lib/thrift/server/base_server.rb +lib/thrift/server/mongrel_http_server.rb +lib/thrift/server/nonblocking_server.rb +lib/thrift/server/simple_server.rb +lib/thrift/server/thread_pool_server.rb +lib/thrift/server/threaded_server.rb +lib/thrift/transport/base_server_transport.rb +lib/thrift/transport/base_transport.rb +lib/thrift/transport/buffered_transport.rb +lib/thrift/transport/framed_transport.rb +lib/thrift/transport/http_client_transport.rb +lib/thrift/transport/io_stream_transport.rb +lib/thrift/transport/memory_buffer_transport.rb +lib/thrift/transport/server_socket.rb +lib/thrift/transport/socket.rb +lib/thrift/transport/unix_server_socket.rb +lib/thrift/transport/unix_socket.rb +script/proto_benchmark.rb +script/read_struct.rb +script/write_struct.rb +spec/base_protocol_spec.rb +spec/base_transport_spec.rb +spec/binary_protocol_accelerated_spec.rb +spec/binary_protocol_spec.rb +spec/binary_protocol_spec_shared.rb +spec/client_spec.rb +spec/compact_protocol_spec.rb +spec/exception_spec.rb +spec/http_client_spec.rb +spec/mongrel_http_server_spec.rb +spec/nonblocking_server_spec.rb +spec/processor_spec.rb +spec/serializer_spec.rb +spec/server_socket_spec.rb +spec/server_spec.rb +spec/socket_spec.rb +spec/socket_spec_shared.rb +spec/spec_helper.rb +spec/struct_spec.rb +spec/ThriftSpec.thrift +spec/types_spec.rb +spec/unix_socket_spec.rb diff --git a/lib/rb/README b/lib/rb/README new file mode 100644 index 00000000..d78e3527 --- /dev/null +++ b/lib/rb/README @@ -0,0 +1,43 @@ +Thrift Ruby Software Library + http://incubator.apache.org/thrift/ + +== LICENSE: + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +== DESCRIPTION: + +Thrift is a strongly-typed language-agnostic RPC system. +This library is the ruby implementation for both clients and servers. + +== INSTALL: + + $ gem install thrift + +== CAVEATS: + +This library provides the client and server implementations of thrift. +It does not provide the compiler for the .thrift files. To compile +.thrift files into language-specific implementations, please download the full +thrift software package. + +== USAGE: + +This section should get written by someone with the time and inclination. +In the meantime, look at existing code, such as the benchmark or the tutorial +in the full thrift distribution. diff --git a/lib/rb/Rakefile b/lib/rb/Rakefile new file mode 100644 index 00000000..1a9467a5 --- /dev/null +++ b/lib/rb/Rakefile @@ -0,0 +1,103 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'rubygems' +require 'rake' +require 'spec/rake/spectask' + +THRIFT = '../../compiler/cpp/thrift' + +task :default => [:spec] + +task :spec => [:'gen-rb', :realspec] + +Spec::Rake::SpecTask.new(:realspec) do |t| + t.spec_files = FileList['spec/**/*_spec.rb'] + t.spec_opts = ['--color'] +end + +Spec::Rake::SpecTask.new(:'spec:rcov') do |t| + t.spec_files = FileList['spec/**/*_spec.rb'] + t.spec_opts = ['--color'] + t.rcov = true + t.rcov_opts = ['--exclude', '^spec,/gems/'] +end + +desc 'Run the compiler tests (requires full thrift checkout)' +task :test do + # ensure this is a full thrift checkout and not a tarball of the ruby libs + cmd = 'head -1 ../../README 2>/dev/null | grep Thrift >/dev/null 2>/dev/null' + system(cmd) or fail "rake test requires a full thrift checkout" + sh 'make', '-C', File.dirname(__FILE__) + "/../../test/rb", "check" +end + +desc 'Compile the .thrift files for the specs' +task :'gen-rb' => [:'gen-rb:spec', :'gen-rb:benchmark', :'gen-rb:debug_proto'] + +namespace :'gen-rb' do + task :'spec' do + dir = File.dirname(__FILE__) + '/spec' + sh THRIFT, '--gen', 'rb', '-o', dir, "#{dir}/ThriftSpec.thrift" + end + + task :'benchmark' do + dir = File.dirname(__FILE__) + '/benchmark' + sh THRIFT, '--gen', 'rb', '-o', dir, "#{dir}/Benchmark.thrift" + end + + task :'debug_proto' do + sh "mkdir", "-p", "debug_proto_test" + sh THRIFT, '--gen', 'rb', "-o", "debug_proto_test", "../../test/DebugProtoTest.thrift" + end +end + +desc 'Run benchmarking of NonblockingServer' +task :benchmark do + ruby 'benchmark/benchmark.rb' +end + + +begin + require 'echoe' + + Echoe.new('thrift') do |p| + p.author = ['Kevin Ballard', 'Kevin Clark', 'Mark Slee'] + p.email = ['kevin@sb.org', 'kevin.clark@gmail.com', 'mcslee@facebook.com'] + p.summary = "Ruby libraries for Thrift (a language-agnostic RPC system)" + p.url = "http://incubator.apache.org/thrift/" + p.include_rakefile = true + p.version = "0.1.0" + end + + task :install => [:check_site_lib] + + require 'rbconfig' + task :check_site_lib do + if File.exist?(File.join(Config::CONFIG['sitelibdir'], 'thrift.rb')) + fail "thrift is already installed in site_ruby" + end + end +rescue LoadError + [:install, :package].each do |t| + desc "Stub for #{t}" + task t do + fail "The Echoe gem is required for this task" + end + end +end diff --git a/lib/rb/benchmark/Benchmark.thrift b/lib/rb/benchmark/Benchmark.thrift new file mode 100644 index 00000000..eb5ae38e --- /dev/null +++ b/lib/rb/benchmark/Benchmark.thrift @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +namespace rb ThriftBenchmark + +service BenchmarkService { + i32 fibonacci(1:byte n) +} diff --git a/lib/rb/benchmark/benchmark.rb b/lib/rb/benchmark/benchmark.rb new file mode 100644 index 00000000..3dc67dd8 --- /dev/null +++ b/lib/rb/benchmark/benchmark.rb @@ -0,0 +1,271 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'rubygems' +$:.unshift File.dirname(__FILE__) + '/../lib' +require 'thrift' +require 'stringio' + +HOST = '127.0.0.1' +PORT = 42587 + +############### +## Server +############### + +class Server + attr_accessor :serverclass + attr_accessor :interpreter + attr_accessor :host + attr_accessor :port + + def initialize(opts) + @serverclass = opts.fetch(:class, Thrift::NonblockingServer) + @interpreter = opts.fetch(:interpreter, "ruby") + @host = opts.fetch(:host, ::HOST) + @port = opts.fetch(:port, ::PORT) + end + + def start + return if @serverclass == Object + args = (File.basename(@interpreter) == "jruby" ? "-J-server" : "") + @pipe = IO.popen("#{@interpreter} #{args} #{File.dirname(__FILE__)}/server.rb #{@host} #{@port} #{@serverclass.name}", "r+") + Marshal.load(@pipe) # wait until the server has started + sleep 0.4 # give the server time to actually start spawning sockets + end + + def shutdown + return unless @pipe + Marshal.dump(:shutdown, @pipe) + begin + @pipe.read(10) # block until the server shuts down + rescue EOFError + end + @pipe.close + @pipe = nil + end +end + +class BenchmarkManager + def initialize(opts, server) + @socket = opts.fetch(:socket) do + @host = opts.fetch(:host, 'localhost') + @port = opts.fetch(:port) + nil + end + @num_processes = opts.fetch(:num_processes, 40) + @clients_per_process = opts.fetch(:clients_per_process, 10) + @calls_per_client = opts.fetch(:calls_per_client, 50) + @interpreter = opts.fetch(:interpreter, "ruby") + @server = server + @log_exceptions = opts.fetch(:log_exceptions, false) + end + + def run + @pool = [] + @benchmark_start = Time.now + puts "Spawning benchmark processes..." + @num_processes.times do + spawn + sleep 0.02 # space out spawns + end + collect_output + @benchmark_end = Time.now # we know the procs are done here + translate_output + analyze_output + report_output + end + + def spawn + pipe = IO.popen("#{@interpreter} #{File.dirname(__FILE__)}/client.rb #{"-log-exceptions" if @log_exceptions} #{@host} #{@port} #{@clients_per_process} #{@calls_per_client}") + @pool << pipe + end + + def socket_class + if @socket + Thrift::UNIXSocket + else + Thrift::Socket + end + end + + def collect_output + puts "Collecting output..." + # read from @pool until all sockets are closed + @buffers = Hash.new { |h,k| h[k] = '' } + until @pool.empty? + rd, = select(@pool) + next if rd.nil? + rd.each do |fd| + begin + @buffers[fd] << fd.readpartial(4096) + rescue EOFError + @pool.delete fd + end + end + end + end + + def translate_output + puts "Translating output..." + @output = [] + @buffers.each do |fd, buffer| + strio = StringIO.new(buffer) + logs = [] + begin + loop do + logs << Marshal.load(strio) + end + rescue EOFError + @output << logs + end + end + end + + def analyze_output + puts "Analyzing output..." + call_times = [] + client_times = [] + connection_failures = [] + connection_errors = [] + shortest_call = 0 + shortest_client = 0 + longest_call = 0 + longest_client = 0 + @output.each do |logs| + cur_call, cur_client = nil + logs.each do |tok, time| + case tok + when :start + cur_client = time + when :call_start + cur_call = time + when :call_end + delta = time - cur_call + call_times << delta + longest_call = delta unless longest_call > delta + shortest_call = delta if shortest_call == 0 or delta < shortest_call + cur_call = nil + when :end + delta = time - cur_client + client_times << delta + longest_client = delta unless longest_client > delta + shortest_client = delta if shortest_client == 0 or delta < shortest_client + cur_client = nil + when :connection_failure + connection_failures << time + when :connection_error + connection_errors << time + end + end + end + @report = {} + @report[:total_calls] = call_times.inject(0.0) { |a,t| a += t } + @report[:avg_calls] = @report[:total_calls] / call_times.size + @report[:total_clients] = client_times.inject(0.0) { |a,t| a += t } + @report[:avg_clients] = @report[:total_clients] / client_times.size + @report[:connection_failures] = connection_failures.size + @report[:connection_errors] = connection_errors.size + @report[:shortest_call] = shortest_call + @report[:shortest_client] = shortest_client + @report[:longest_call] = longest_call + @report[:longest_client] = longest_client + @report[:total_benchmark_time] = @benchmark_end - @benchmark_start + @report[:fastthread] = $".include?('fastthread.bundle') + end + + def report_output + fmt = "%.4f seconds" + puts + tabulate "%d", + [["Server class", "%s"], @server.serverclass == Object ? "" : @server.serverclass], + [["Server interpreter", "%s"], @server.interpreter], + [["Client interpreter", "%s"], @interpreter], + [["Socket class", "%s"], socket_class], + ["Number of processes", @num_processes], + ["Clients per process", @clients_per_process], + ["Calls per client", @calls_per_client], + [["Using fastthread", "%s"], @report[:fastthread] ? "yes" : "no"] + puts + failures = (@report[:connection_failures] > 0) + tabulate fmt, + [["Connection failures", "%d", [:red, :bold]], @report[:connection_failures]], + [["Connection errors", "%d", [:red, :bold]], @report[:connection_errors]], + ["Average time per call", @report[:avg_calls]], + ["Average time per client (%d calls)" % @calls_per_client, @report[:avg_clients]], + ["Total time for all calls", @report[:total_calls]], + ["Real time for benchmarking", @report[:total_benchmark_time]], + ["Shortest call time", @report[:shortest_call]], + ["Longest call time", @report[:longest_call]], + ["Shortest client time (%d calls)" % @calls_per_client, @report[:shortest_client]], + ["Longest client time (%d calls)" % @calls_per_client, @report[:longest_client]] + end + + ANSI = { + :reset => 0, + :bold => 1, + :black => 30, + :red => 31, + :green => 32, + :yellow => 33, + :blue => 34, + :magenta => 35, + :cyan => 36, + :white => 37 + } + + def tabulate(fmt, *labels_and_values) + labels = labels_and_values.map { |l| Array === l ? l.first : l } + label_width = labels.inject(0) { |w,l| l.size > w ? l.size : w } + labels_and_values.each do |(l,v)| + f = fmt + l, f, c = l if Array === l + fmtstr = "%-#{label_width+1}s #{f}" + if STDOUT.tty? and c and v.to_i > 0 + fmtstr = "\e[#{[*c].map { |x| ANSI[x] } * ";"}m" + fmtstr + "\e[#{ANSI[:reset]}m" + end + puts fmtstr % [l+":", v] + end + end +end + +def resolve_const(const) + const and const.split('::').inject(Object) { |k,c| k.const_get(c) } +end + +puts "Starting server..." +args = {} +args[:interpreter] = ENV['THRIFT_SERVER_INTERPRETER'] || ENV['THRIFT_INTERPRETER'] || "ruby" +args[:class] = resolve_const(ENV['THRIFT_SERVER']) || Thrift::NonblockingServer +args[:host] = ENV['THRIFT_HOST'] || HOST +args[:port] = (ENV['THRIFT_PORT'] || PORT).to_i +server = Server.new(args) +server.start + +args = {} +args[:host] = ENV['THRIFT_HOST'] || HOST +args[:port] = (ENV['THRIFT_PORT'] || PORT).to_i +args[:num_processes] = (ENV['THRIFT_NUM_PROCESSES'] || 40).to_i +args[:clients_per_process] = (ENV['THRIFT_NUM_CLIENTS'] || 5).to_i +args[:calls_per_client] = (ENV['THRIFT_NUM_CALLS'] || 50).to_i +args[:interpreter] = ENV['THRIFT_CLIENT_INTERPRETER'] || ENV['THRIFT_INTERPRETER'] || "ruby" +args[:log_exceptions] = !!ENV['THRIFT_LOG_EXCEPTIONS'] +BenchmarkManager.new(args, server).run + +server.shutdown diff --git a/lib/rb/benchmark/client.rb b/lib/rb/benchmark/client.rb new file mode 100644 index 00000000..703dc8f5 --- /dev/null +++ b/lib/rb/benchmark/client.rb @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +$:.unshift File.dirname(__FILE__) + '/../lib' +require 'thrift' +$:.unshift File.dirname(__FILE__) + "/gen-rb" +require 'benchmark_service' + +class Client + def initialize(host, port, clients_per_process, calls_per_client, log_exceptions) + @host = host + @port = port + @clients_per_process = clients_per_process + @calls_per_client = calls_per_client + @log_exceptions = log_exceptions + end + + def run + @clients_per_process.times do + socket = Thrift::Socket.new(@host, @port) + transport = Thrift::FramedTransport.new(socket) + protocol = Thrift::BinaryProtocol.new(transport) + client = ThriftBenchmark::BenchmarkService::Client.new(protocol) + begin + start = Time.now + transport.open + Marshal.dump [:start, start], STDOUT + rescue => e + Marshal.dump [:connection_failure, Time.now], STDOUT + print_exception e if @log_exceptions + else + begin + @calls_per_client.times do + Marshal.dump [:call_start, Time.now], STDOUT + client.fibonacci(15) + Marshal.dump [:call_end, Time.now], STDOUT + end + transport.close + Marshal.dump [:end, Time.now], STDOUT + rescue Thrift::TransportException => e + Marshal.dump [:connection_error, Time.now], STDOUT + print_exception e if @log_exceptions + end + end + end + end + + def print_exception(e) + STDERR.puts "ERROR: #{e.message}" + STDERR.puts "\t#{e.backtrace * "\n\t"}" + end +end + +log_exceptions = true if ARGV[0] == '-log-exceptions' and ARGV.shift + +host, port, clients_per_process, calls_per_client = ARGV + +Client.new(host, port.to_i, clients_per_process.to_i, calls_per_client.to_i, log_exceptions).run diff --git a/lib/rb/benchmark/server.rb b/lib/rb/benchmark/server.rb new file mode 100644 index 00000000..74e13f41 --- /dev/null +++ b/lib/rb/benchmark/server.rb @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +$:.unshift File.dirname(__FILE__) + '/../lib' +require 'thrift' +$:.unshift File.dirname(__FILE__) + "/gen-rb" +require 'benchmark_service' + +module Server + include Thrift + + class BenchmarkHandler + # 1-based index into the fibonacci sequence + def fibonacci(n) + seq = [1, 1] + 3.upto(n) do + seq << seq[-1] + seq[-2] + end + seq[n-1] # n is 1-based + end + end + + def self.start_server(host, port, serverClass) + handler = BenchmarkHandler.new + processor = ThriftBenchmark::BenchmarkService::Processor.new(handler) + transport = ServerSocket.new(host, port) + transport_factory = FramedTransportFactory.new + args = [processor, transport, transport_factory, nil, 20] + if serverClass == NonblockingServer + logger = Logger.new(STDERR) + logger.level = Logger::WARN + args << logger + end + server = serverClass.new(*args) + @server_thread = Thread.new do + server.serve + end + @server = server + end + + def self.shutdown + return if @server.nil? + if @server.respond_to? :shutdown + @server.shutdown + else + @server_thread.kill + end + end +end + +def resolve_const(const) + const and const.split('::').inject(Object) { |k,c| k.const_get(c) } +end + +host, port, serverklass = ARGV + +Server.start_server(host, port.to_i, resolve_const(serverklass)) + +# let our host know that the interpreter has started +# ideally we'd wait until the server was serving, but we don't have a hook for that +Marshal.dump(:started, STDOUT) +STDOUT.flush + +Marshal.load(STDIN) # wait until we're instructed to shut down + +Server.shutdown diff --git a/lib/rb/benchmark/thin_server.rb b/lib/rb/benchmark/thin_server.rb new file mode 100644 index 00000000..4de2eef3 --- /dev/null +++ b/lib/rb/benchmark/thin_server.rb @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +$:.unshift File.dirname(__FILE__) + '/../lib' +require 'thrift' +$:.unshift File.dirname(__FILE__) + "/gen-rb" +require 'benchmark_service' +HOST = 'localhost' +PORT = 42587 + +class BenchmarkHandler + # 1-based index into the fibonacci sequence + def fibonacci(n) + seq = [1, 1] + 3.upto(n) do + seq << seq[-1] + seq[-2] + end + seq[n-1] # n is 1-based + end +end + +handler = BenchmarkHandler.new +processor = ThriftBenchmark::BenchmarkService::Processor.new(handler) +transport = Thrift::ServerSocket.new(HOST, PORT) +transport_factory = Thrift::FramedTransportFactory.new +logger = Logger.new(STDERR) +logger.level = Logger::WARN +Thrift::NonblockingServer.new(processor, transport, transport_factory, nil, 20, logger).serve diff --git a/lib/rb/ext/binary_protocol_accelerated.c b/lib/rb/ext/binary_protocol_accelerated.c new file mode 100644 index 00000000..728a0572 --- /dev/null +++ b/lib/rb/ext/binary_protocol_accelerated.c @@ -0,0 +1,474 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include "macros.h" + +VALUE rb_thrift_binary_proto_native_qmark(VALUE self) { + return Qtrue; +} + + + +static int VERSION_1; +static int VERSION_MASK; +static int TYPE_MASK; +static int BAD_VERSION; + +static void write_byte_direct(VALUE trans, int8_t b) { + WRITE(trans, (char*)&b, 1); +} + +static void write_i16_direct(VALUE trans, int16_t value) { + char data[2]; + + data[1] = value; + data[0] = (value >> 8); + + WRITE(trans, data, 2); +} + +static void write_i32_direct(VALUE trans, int32_t value) { + char data[4]; + + data[3] = value; + data[2] = (value >> 8); + data[1] = (value >> 16); + data[0] = (value >> 24); + + WRITE(trans, data, 4); +} + + +static void write_i64_direct(VALUE trans, int64_t value) { + char data[8]; + + data[7] = value; + data[6] = (value >> 8); + data[5] = (value >> 16); + data[4] = (value >> 24); + data[3] = (value >> 32); + data[2] = (value >> 40); + data[1] = (value >> 48); + data[0] = (value >> 56); + + WRITE(trans, data, 8); +} + +static void write_string_direct(VALUE trans, VALUE str) { + write_i32_direct(trans, RSTRING_LEN(str)); + rb_funcall(trans, write_method_id, 1, str); +} + +//-------------------------------- +// interface writing methods +//-------------------------------- + +VALUE rb_thrift_binary_proto_write_message_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_struct_begin(VALUE self, VALUE name) { + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_struct_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_field_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_map_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_list_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_set_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_message_begin(VALUE self, VALUE name, VALUE type, VALUE seqid) { + VALUE trans = GET_TRANSPORT(self); + VALUE strict_write = GET_STRICT_WRITE(self); + + if (strict_write == Qtrue) { + write_i32_direct(trans, VERSION_1 | FIX2INT(type)); + write_string_direct(trans, name); + write_i32_direct(trans, FIX2INT(seqid)); + } else { + write_string_direct(trans, name); + write_byte_direct(trans, FIX2INT(type)); + write_i32_direct(trans, FIX2INT(seqid)); + } + + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_field_begin(VALUE self, VALUE name, VALUE type, VALUE id) { + VALUE trans = GET_TRANSPORT(self); + write_byte_direct(trans, FIX2INT(type)); + write_i16_direct(trans, FIX2INT(id)); + + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_field_stop(VALUE self) { + write_byte_direct(GET_TRANSPORT(self), TTYPE_STOP); + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_map_begin(VALUE self, VALUE ktype, VALUE vtype, VALUE size) { + VALUE trans = GET_TRANSPORT(self); + write_byte_direct(trans, FIX2INT(ktype)); + write_byte_direct(trans, FIX2INT(vtype)); + write_i32_direct(trans, FIX2INT(size)); + + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_list_begin(VALUE self, VALUE etype, VALUE size) { + VALUE trans = GET_TRANSPORT(self); + write_byte_direct(trans, FIX2INT(etype)); + write_i32_direct(trans, FIX2INT(size)); + + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_set_begin(VALUE self, VALUE etype, VALUE size) { + rb_thrift_binary_proto_write_list_begin(self, etype, size); + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_bool(VALUE self, VALUE b) { + write_byte_direct(GET_TRANSPORT(self), RTEST(b) ? 1 : 0); + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_byte(VALUE self, VALUE byte) { + CHECK_NIL(byte); + write_byte_direct(GET_TRANSPORT(self), NUM2INT(byte)); + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_i16(VALUE self, VALUE i16) { + CHECK_NIL(i16); + write_i16_direct(GET_TRANSPORT(self), FIX2INT(i16)); + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_i32(VALUE self, VALUE i32) { + CHECK_NIL(i32); + write_i32_direct(GET_TRANSPORT(self), NUM2INT(i32)); + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_i64(VALUE self, VALUE i64) { + CHECK_NIL(i64); + write_i64_direct(GET_TRANSPORT(self), NUM2LL(i64)); + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_double(VALUE self, VALUE dub) { + CHECK_NIL(dub); + // Unfortunately, bitwise_cast doesn't work in C. Bad C! + union { + double f; + int64_t t; + } transfer; + transfer.f = RFLOAT_VALUE(rb_Float(dub)); + write_i64_direct(GET_TRANSPORT(self), transfer.t); + + return Qnil; +} + +VALUE rb_thrift_binary_proto_write_string(VALUE self, VALUE str) { + CHECK_NIL(str); + VALUE trans = GET_TRANSPORT(self); + write_string_direct(trans, str); + return Qnil; +} + +//--------------------------------------- +// interface reading methods +//--------------------------------------- + +VALUE rb_thrift_binary_proto_read_string(VALUE self); +VALUE rb_thrift_binary_proto_read_byte(VALUE self); +VALUE rb_thrift_binary_proto_read_i32(VALUE self); +VALUE rb_thrift_binary_proto_read_i16(VALUE self); + +static char read_byte_direct(VALUE self) { + VALUE buf = READ(self, 1); + return RSTRING_PTR(buf)[0]; +} + +static int16_t read_i16_direct(VALUE self) { + VALUE buf = READ(self, 2); + return (int16_t)(((uint8_t)(RSTRING_PTR(buf)[1])) | ((uint16_t)((RSTRING_PTR(buf)[0]) << 8))); +} + +static int32_t read_i32_direct(VALUE self) { + VALUE buf = READ(self, 4); + return ((uint8_t)(RSTRING_PTR(buf)[3])) | + (((uint8_t)(RSTRING_PTR(buf)[2])) << 8) | + (((uint8_t)(RSTRING_PTR(buf)[1])) << 16) | + (((uint8_t)(RSTRING_PTR(buf)[0])) << 24); +} + +static int64_t read_i64_direct(VALUE self) { + uint64_t hi = read_i32_direct(self); + uint32_t lo = read_i32_direct(self); + return (hi << 32) | lo; +} + +static VALUE get_protocol_exception(VALUE code, VALUE message) { + VALUE args[2]; + args[0] = code; + args[1] = message; + return rb_class_new_instance(2, (VALUE*)&args, protocol_exception_class); +} + +VALUE rb_thrift_binary_proto_read_message_end(VALUE self) { + return Qnil; +} + +VALUE rb_thift_binary_proto_read_struct_begin(VALUE self) { + return Qnil; +} + +VALUE rb_thift_binary_proto_read_struct_end(VALUE self) { + return Qnil; +} + +VALUE rb_thift_binary_proto_read_field_end(VALUE self) { + return Qnil; +} + +VALUE rb_thift_binary_proto_read_map_end(VALUE self) { + return Qnil; +} + +VALUE rb_thift_binary_proto_read_list_end(VALUE self) { + return Qnil; +} + +VALUE rb_thift_binary_proto_read_set_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_binary_proto_read_message_begin(VALUE self) { + VALUE strict_read = GET_STRICT_READ(self); + VALUE name, seqid; + int type; + + int version = read_i32_direct(self); + + if (version < 0) { + if ((version & VERSION_MASK) != VERSION_1) { + rb_exc_raise(get_protocol_exception(INT2FIX(BAD_VERSION), rb_str_new2("Missing version identifier"))); + } + type = version & TYPE_MASK; + name = rb_thrift_binary_proto_read_string(self); + seqid = rb_thrift_binary_proto_read_i32(self); + } else { + if (strict_read == Qtrue) { + rb_exc_raise(get_protocol_exception(INT2FIX(BAD_VERSION), rb_str_new2("No version identifier, old protocol client?"))); + } + name = READ(self, version); + type = read_byte_direct(self); + seqid = rb_thrift_binary_proto_read_i32(self); + } + + return rb_ary_new3(3, name, INT2FIX(type), seqid); +} + +VALUE rb_thrift_binary_proto_read_field_begin(VALUE self) { + int type = read_byte_direct(self); + if (type == TTYPE_STOP) { + return rb_ary_new3(3, Qnil, INT2FIX(type), INT2FIX(0)); + } else { + VALUE id = rb_thrift_binary_proto_read_i16(self); + return rb_ary_new3(3, Qnil, INT2FIX(type), id); + } +} + +VALUE rb_thrift_binary_proto_read_map_begin(VALUE self) { + VALUE ktype = rb_thrift_binary_proto_read_byte(self); + VALUE vtype = rb_thrift_binary_proto_read_byte(self); + VALUE size = rb_thrift_binary_proto_read_i32(self); + return rb_ary_new3(3, ktype, vtype, size); +} + +VALUE rb_thrift_binary_proto_read_list_begin(VALUE self) { + VALUE etype = rb_thrift_binary_proto_read_byte(self); + VALUE size = rb_thrift_binary_proto_read_i32(self); + return rb_ary_new3(2, etype, size); +} + +VALUE rb_thrift_binary_proto_read_set_begin(VALUE self) { + return rb_thrift_binary_proto_read_list_begin(self); +} + +VALUE rb_thrift_binary_proto_read_bool(VALUE self) { + char byte = read_byte_direct(self); + return byte != 0 ? Qtrue : Qfalse; +} + +VALUE rb_thrift_binary_proto_read_byte(VALUE self) { + return INT2FIX(read_byte_direct(self)); +} + +VALUE rb_thrift_binary_proto_read_i16(VALUE self) { + return INT2FIX(read_i16_direct(self)); +} + +VALUE rb_thrift_binary_proto_read_i32(VALUE self) { + return INT2NUM(read_i32_direct(self)); +} + +VALUE rb_thrift_binary_proto_read_i64(VALUE self) { + return LL2NUM(read_i64_direct(self)); +} + +VALUE rb_thrift_binary_proto_read_double(VALUE self) { + union { + double f; + int64_t t; + } transfer; + transfer.t = read_i64_direct(self); + return rb_float_new(transfer.f); +} + +VALUE rb_thrift_binary_proto_read_string(VALUE self) { + int size = read_i32_direct(self); + return READ(self, size); +} + +void Init_binary_protocol_accelerated() { + VALUE thrift_binary_protocol_class = rb_const_get(thrift_module, rb_intern("BinaryProtocol")); + + VERSION_1 = rb_num2ll(rb_const_get(thrift_binary_protocol_class, rb_intern("VERSION_1"))); + VERSION_MASK = rb_num2ll(rb_const_get(thrift_binary_protocol_class, rb_intern("VERSION_MASK"))); + TYPE_MASK = rb_num2ll(rb_const_get(thrift_binary_protocol_class, rb_intern("TYPE_MASK"))); + + VALUE bpa_class = rb_define_class_under(thrift_module, "BinaryProtocolAccelerated", thrift_binary_protocol_class); + + rb_define_method(bpa_class, "native?", rb_thrift_binary_proto_native_qmark, 0); + + rb_define_method(bpa_class, "write_message_begin", rb_thrift_binary_proto_write_message_begin, 3); + rb_define_method(bpa_class, "write_field_begin", rb_thrift_binary_proto_write_field_begin, 3); + rb_define_method(bpa_class, "write_field_stop", rb_thrift_binary_proto_write_field_stop, 0); + rb_define_method(bpa_class, "write_map_begin", rb_thrift_binary_proto_write_map_begin, 3); + rb_define_method(bpa_class, "write_list_begin", rb_thrift_binary_proto_write_list_begin, 2); + rb_define_method(bpa_class, "write_set_begin", rb_thrift_binary_proto_write_set_begin, 2); + rb_define_method(bpa_class, "write_byte", rb_thrift_binary_proto_write_byte, 1); + rb_define_method(bpa_class, "write_bool", rb_thrift_binary_proto_write_bool, 1); + rb_define_method(bpa_class, "write_i16", rb_thrift_binary_proto_write_i16, 1); + rb_define_method(bpa_class, "write_i32", rb_thrift_binary_proto_write_i32, 1); + rb_define_method(bpa_class, "write_i64", rb_thrift_binary_proto_write_i64, 1); + rb_define_method(bpa_class, "write_double", rb_thrift_binary_proto_write_double, 1); + rb_define_method(bpa_class, "write_string", rb_thrift_binary_proto_write_string, 1); + // unused methods + rb_define_method(bpa_class, "write_message_end", rb_thrift_binary_proto_write_message_end, 0); + rb_define_method(bpa_class, "write_struct_begin", rb_thrift_binary_proto_write_struct_begin, 1); + rb_define_method(bpa_class, "write_struct_end", rb_thrift_binary_proto_write_struct_end, 0); + rb_define_method(bpa_class, "write_field_end", rb_thrift_binary_proto_write_field_end, 0); + rb_define_method(bpa_class, "write_map_end", rb_thrift_binary_proto_write_map_end, 0); + rb_define_method(bpa_class, "write_list_end", rb_thrift_binary_proto_write_list_end, 0); + rb_define_method(bpa_class, "write_set_end", rb_thrift_binary_proto_write_set_end, 0); + + + + rb_define_method(bpa_class, "read_message_begin", rb_thrift_binary_proto_read_message_begin, 0); + rb_define_method(bpa_class, "read_field_begin", rb_thrift_binary_proto_read_field_begin, 0); + rb_define_method(bpa_class, "read_map_begin", rb_thrift_binary_proto_read_map_begin, 0); + rb_define_method(bpa_class, "read_list_begin", rb_thrift_binary_proto_read_list_begin, 0); + rb_define_method(bpa_class, "read_set_begin", rb_thrift_binary_proto_read_set_begin, 0); + rb_define_method(bpa_class, "read_byte", rb_thrift_binary_proto_read_byte, 0); + rb_define_method(bpa_class, "read_bool", rb_thrift_binary_proto_read_bool, 0); + rb_define_method(bpa_class, "read_i16", rb_thrift_binary_proto_read_i16, 0); + rb_define_method(bpa_class, "read_i32", rb_thrift_binary_proto_read_i32, 0); + rb_define_method(bpa_class, "read_i64", rb_thrift_binary_proto_read_i64, 0); + rb_define_method(bpa_class, "read_double", rb_thrift_binary_proto_read_double, 0); + rb_define_method(bpa_class, "read_string", rb_thrift_binary_proto_read_string, 0); + // unused methods + rb_define_method(bpa_class, "read_message_end", rb_thrift_binary_proto_read_message_end, 0); + rb_define_method(bpa_class, "read_struct_begin", rb_thift_binary_proto_read_struct_begin, 0); + rb_define_method(bpa_class, "read_struct_end", rb_thift_binary_proto_read_struct_end, 0); + rb_define_method(bpa_class, "read_field_end", rb_thift_binary_proto_read_field_end, 0); + rb_define_method(bpa_class, "read_map_end", rb_thift_binary_proto_read_map_end, 0); + rb_define_method(bpa_class, "read_list_end", rb_thift_binary_proto_read_list_end, 0); + rb_define_method(bpa_class, "read_set_end", rb_thift_binary_proto_read_set_end, 0); + + // set up native method table + native_proto_method_table *npmt; + npmt = ALLOC(native_proto_method_table); + + npmt->write_field_begin = rb_thrift_binary_proto_write_field_begin; + npmt->write_field_stop = rb_thrift_binary_proto_write_field_stop; + npmt->write_map_begin = rb_thrift_binary_proto_write_map_begin; + npmt->write_list_begin = rb_thrift_binary_proto_write_list_begin; + npmt->write_set_begin = rb_thrift_binary_proto_write_set_begin; + npmt->write_byte = rb_thrift_binary_proto_write_byte; + npmt->write_bool = rb_thrift_binary_proto_write_bool; + npmt->write_i16 = rb_thrift_binary_proto_write_i16; + npmt->write_i32 = rb_thrift_binary_proto_write_i32; + npmt->write_i64 = rb_thrift_binary_proto_write_i64; + npmt->write_double = rb_thrift_binary_proto_write_double; + npmt->write_string = rb_thrift_binary_proto_write_string; + npmt->write_message_end = rb_thrift_binary_proto_write_message_end; + npmt->write_struct_begin = rb_thrift_binary_proto_write_struct_begin; + npmt->write_struct_end = rb_thrift_binary_proto_write_struct_end; + npmt->write_field_end = rb_thrift_binary_proto_write_field_end; + npmt->write_map_end = rb_thrift_binary_proto_write_map_end; + npmt->write_list_end = rb_thrift_binary_proto_write_list_end; + npmt->write_set_end = rb_thrift_binary_proto_write_set_end; + + npmt->read_message_begin = rb_thrift_binary_proto_read_message_begin; + npmt->read_field_begin = rb_thrift_binary_proto_read_field_begin; + npmt->read_map_begin = rb_thrift_binary_proto_read_map_begin; + npmt->read_list_begin = rb_thrift_binary_proto_read_list_begin; + npmt->read_set_begin = rb_thrift_binary_proto_read_set_begin; + npmt->read_byte = rb_thrift_binary_proto_read_byte; + npmt->read_bool = rb_thrift_binary_proto_read_bool; + npmt->read_i16 = rb_thrift_binary_proto_read_i16; + npmt->read_i32 = rb_thrift_binary_proto_read_i32; + npmt->read_i64 = rb_thrift_binary_proto_read_i64; + npmt->read_double = rb_thrift_binary_proto_read_double; + npmt->read_string = rb_thrift_binary_proto_read_string; + npmt->read_message_end = rb_thrift_binary_proto_read_message_end; + npmt->read_struct_begin = rb_thift_binary_proto_read_struct_begin; + npmt->read_struct_end = rb_thift_binary_proto_read_struct_end; + npmt->read_field_end = rb_thift_binary_proto_read_field_end; + npmt->read_map_end = rb_thift_binary_proto_read_map_end; + npmt->read_list_end = rb_thift_binary_proto_read_list_end; + npmt->read_set_end = rb_thift_binary_proto_read_set_end; + + VALUE method_table_object = Data_Wrap_Struct(rb_cObject, 0, free, npmt); + rb_const_set(bpa_class, rb_intern("@native_method_table"), method_table_object); +} diff --git a/lib/rb/ext/binary_protocol_accelerated.h b/lib/rb/ext/binary_protocol_accelerated.h new file mode 100644 index 00000000..37baf414 --- /dev/null +++ b/lib/rb/ext/binary_protocol_accelerated.h @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +void Init_binary_protocol_accelerated(); diff --git a/lib/rb/ext/compact_protocol.c b/lib/rb/ext/compact_protocol.c new file mode 100644 index 00000000..7966d3e3 --- /dev/null +++ b/lib/rb/ext/compact_protocol.c @@ -0,0 +1,665 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include "macros.h" + +#define LAST_ID(obj) FIX2INT(rb_ary_pop(rb_ivar_get(obj, last_field_id))) +#define SET_LAST_ID(obj, val) rb_ary_push(rb_ivar_get(obj, last_field_id), val) + +VALUE rb_thrift_compact_proto_native_qmark(VALUE self) { + return Qtrue; +} + +static ID last_field_id; +static ID boolean_field_id; +static ID bool_value_id; + +static int VERSION; +static int VERSION_MASK; +static int TYPE_MASK; +static int TYPE_SHIFT_AMOUNT; +static int PROTOCOL_ID; + +static VALUE thrift_compact_protocol_class; + +static int CTYPE_BOOLEAN_TRUE = 0x01; +static int CTYPE_BOOLEAN_FALSE = 0x02; +static int CTYPE_BYTE = 0x03; +static int CTYPE_I16 = 0x04; +static int CTYPE_I32 = 0x05; +static int CTYPE_I64 = 0x06; +static int CTYPE_DOUBLE = 0x07; +static int CTYPE_BINARY = 0x08; +static int CTYPE_LIST = 0x09; +static int CTYPE_SET = 0x0A; +static int CTYPE_MAP = 0x0B; +static int CTYPE_STRUCT = 0x0C; + +VALUE rb_thrift_compact_proto_write_i16(VALUE self, VALUE i16); + +// TODO: implement this +static int get_compact_type(VALUE type_value) { + int type = FIX2INT(type_value); + if (type == TTYPE_BOOL) { + return CTYPE_BOOLEAN_TRUE; + } else if (type == TTYPE_BYTE) { + return CTYPE_BYTE; + } else if (type == TTYPE_I16) { + return CTYPE_I16; + } else if (type == TTYPE_I32) { + return CTYPE_I32; + } else if (type == TTYPE_I64) { + return CTYPE_I64; + } else if (type == TTYPE_DOUBLE) { + return CTYPE_DOUBLE; + } else if (type == TTYPE_STRING) { + return CTYPE_BINARY; + } else if (type == TTYPE_LIST) { + return CTYPE_LIST; + } else if (type == TTYPE_SET) { + return CTYPE_SET; + } else if (type == TTYPE_MAP) { + return CTYPE_MAP; + } else if (type == TTYPE_STRUCT) { + return CTYPE_STRUCT; + } else { + char str[50]; + sprintf(str, "don't know what type: %d", type); + rb_raise(rb_eStandardError, str); + return 0; + } +} + +static void write_byte_direct(VALUE transport, int8_t b) { + WRITE(transport, (char*)&b, 1); +} + +static void write_field_begin_internal(VALUE self, VALUE type, VALUE id_value, VALUE type_override) { + int id = FIX2INT(id_value); + int last_id = LAST_ID(self); + VALUE transport = GET_TRANSPORT(self); + + // if there's a type override, use that. + int8_t type_to_write = RTEST(type_override) ? FIX2INT(type_override) : get_compact_type(type); + // check if we can use delta encoding for the field id + int diff = id - last_id; + if (diff > 0 && diff <= 15) { + // write them together + write_byte_direct(transport, diff << 4 | (type_to_write & 0x0f)); + } else { + // write them separate + write_byte_direct(transport, type_to_write & 0x0f); + rb_thrift_compact_proto_write_i16(self, id_value); + } + + SET_LAST_ID(self, id_value); +} + +static int32_t int_to_zig_zag(int32_t n) { + return (n << 1) ^ (n >> 31); +} + +static uint64_t ll_to_zig_zag(int64_t n) { + return (n << 1) ^ (n >> 63); +} + +static void write_varint32(VALUE transport, uint32_t n) { + while (true) { + if ((n & ~0x7F) == 0) { + write_byte_direct(transport, n & 0x7f); + break; + } else { + write_byte_direct(transport, (n & 0x7F) | 0x80); + n = n >> 7; + } + } +} + +static void write_varint64(VALUE transport, uint64_t n) { + while (true) { + if ((n & ~0x7F) == 0) { + write_byte_direct(transport, n & 0x7f); + break; + } else { + write_byte_direct(transport, (n & 0x7F) | 0x80); + n = n >> 7; + } + } +} + +static void write_collection_begin(VALUE transport, VALUE elem_type, VALUE size_value) { + int size = FIX2INT(size_value); + if (size <= 14) { + write_byte_direct(transport, size << 4 | get_compact_type(elem_type)); + } else { + write_byte_direct(transport, 0xf0 | get_compact_type(elem_type)); + write_varint32(transport, size); + } +} + + +//-------------------------------- +// interface writing methods +//-------------------------------- + +VALUE rb_thrift_compact_proto_write_i32(VALUE self, VALUE i32); +VALUE rb_thrift_compact_proto_write_string(VALUE self, VALUE str); + +VALUE rb_thrift_compact_proto_write_message_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_struct_begin(VALUE self, VALUE name) { + rb_ary_push(rb_ivar_get(self, last_field_id), INT2FIX(0)); + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_struct_end(VALUE self) { + rb_ary_pop(rb_ivar_get(self, last_field_id)); + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_field_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_map_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_list_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_set_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_message_begin(VALUE self, VALUE name, VALUE type, VALUE seqid) { + VALUE transport = GET_TRANSPORT(self); + write_byte_direct(transport, PROTOCOL_ID); + write_byte_direct(transport, (VERSION & VERSION_MASK) | ((FIX2INT(type) << TYPE_SHIFT_AMOUNT) & TYPE_MASK)); + write_varint32(transport, FIX2INT(seqid)); + rb_thrift_compact_proto_write_string(self, name); + + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_field_begin(VALUE self, VALUE name, VALUE type, VALUE id) { + if (FIX2INT(type) == TTYPE_BOOL) { + // we want to possibly include the value, so we'll wait. + rb_ivar_set(self, boolean_field_id, rb_ary_new3(2, type, id)); + } else { + write_field_begin_internal(self, type, id, Qnil); + } + + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_field_stop(VALUE self) { + write_byte_direct(GET_TRANSPORT(self), TTYPE_STOP); + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_map_begin(VALUE self, VALUE ktype, VALUE vtype, VALUE size_value) { + int size = FIX2INT(size_value); + VALUE transport = GET_TRANSPORT(self); + if (size == 0) { + write_byte_direct(transport, 0); + } else { + write_varint32(transport, size); + write_byte_direct(transport, get_compact_type(ktype) << 4 | get_compact_type(vtype)); + } + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_list_begin(VALUE self, VALUE etype, VALUE size) { + write_collection_begin(GET_TRANSPORT(self), etype, size); + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_set_begin(VALUE self, VALUE etype, VALUE size) { + write_collection_begin(GET_TRANSPORT(self), etype, size); + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_bool(VALUE self, VALUE b) { + int8_t type = b == Qtrue ? CTYPE_BOOLEAN_TRUE : CTYPE_BOOLEAN_FALSE; + VALUE boolean_field = rb_ivar_get(self, boolean_field_id); + if (NIL_P(boolean_field)) { + // we're not part of a field, so just write the value. + write_byte_direct(GET_TRANSPORT(self), type); + } else { + // we haven't written the field header yet + write_field_begin_internal(self, rb_ary_entry(boolean_field, 0), rb_ary_entry(boolean_field, 1), INT2FIX(type)); + rb_ivar_set(self, boolean_field_id, Qnil); + } + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_byte(VALUE self, VALUE byte) { + CHECK_NIL(byte); + write_byte_direct(GET_TRANSPORT(self), FIX2INT(byte)); + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_i16(VALUE self, VALUE i16) { + rb_thrift_compact_proto_write_i32(self, i16); + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_i32(VALUE self, VALUE i32) { + CHECK_NIL(i32); + write_varint32(GET_TRANSPORT(self), int_to_zig_zag(NUM2INT(i32))); + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_i64(VALUE self, VALUE i64) { + CHECK_NIL(i64); + write_varint64(GET_TRANSPORT(self), ll_to_zig_zag(NUM2LL(i64))); + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_double(VALUE self, VALUE dub) { + CHECK_NIL(dub); + // Unfortunately, bitwise_cast doesn't work in C. Bad C! + union { + double f; + int64_t l; + } transfer; + transfer.f = RFLOAT_VALUE(rb_Float(dub)); + char buf[8]; + buf[0] = transfer.l & 0xff; + buf[1] = (transfer.l >> 8) & 0xff; + buf[2] = (transfer.l >> 16) & 0xff; + buf[3] = (transfer.l >> 24) & 0xff; + buf[4] = (transfer.l >> 32) & 0xff; + buf[5] = (transfer.l >> 40) & 0xff; + buf[6] = (transfer.l >> 48) & 0xff; + buf[7] = (transfer.l >> 56) & 0xff; + WRITE(GET_TRANSPORT(self), buf, 8); + return Qnil; +} + +VALUE rb_thrift_compact_proto_write_string(VALUE self, VALUE str) { + VALUE transport = GET_TRANSPORT(self); + write_varint32(transport, RSTRING_LEN(str)); + WRITE(transport, RSTRING_PTR(str), RSTRING_LEN(str)); + return Qnil; +} + +//--------------------------------------- +// interface reading methods +//--------------------------------------- + +#define is_bool_type(ctype) (((ctype) & 0x0F) == CTYPE_BOOLEAN_TRUE || ((ctype) & 0x0F) == CTYPE_BOOLEAN_FALSE) + +VALUE rb_thrift_compact_proto_read_string(VALUE self); +VALUE rb_thrift_compact_proto_read_byte(VALUE self); +VALUE rb_thrift_compact_proto_read_i32(VALUE self); +VALUE rb_thrift_compact_proto_read_i16(VALUE self); + +static int8_t get_ttype(int8_t ctype) { + if (ctype == TTYPE_STOP) { + return TTYPE_STOP; + } else if (ctype == CTYPE_BOOLEAN_TRUE || ctype == CTYPE_BOOLEAN_FALSE) { + return TTYPE_BOOL; + } else if (ctype == CTYPE_BYTE) { + return TTYPE_BYTE; + } else if (ctype == CTYPE_I16) { + return TTYPE_I16; + } else if (ctype == CTYPE_I32) { + return TTYPE_I32; + } else if (ctype == CTYPE_I64) { + return TTYPE_I64; + } else if (ctype == CTYPE_DOUBLE) { + return TTYPE_DOUBLE; + } else if (ctype == CTYPE_BINARY) { + return TTYPE_STRING; + } else if (ctype == CTYPE_LIST) { + return TTYPE_LIST; + } else if (ctype == CTYPE_SET) { + return TTYPE_SET; + } else if (ctype == CTYPE_MAP) { + return TTYPE_MAP; + } else if (ctype == CTYPE_STRUCT) { + return TTYPE_STRUCT; + } else { + char str[50]; + sprintf(str, "don't know what type: %d", ctype); + rb_raise(rb_eStandardError, str); + return 0; + } +} + +static char read_byte_direct(VALUE self) { + VALUE buf = READ(self, 1); + return RSTRING_PTR(buf)[0]; +} + +static int64_t zig_zag_to_ll(int64_t n) { + return (((uint64_t)n) >> 1) ^ -(n & 1); +} + +static int32_t zig_zag_to_int(int32_t n) { + return (((uint32_t)n) >> 1) ^ -(n & 1); +} + +static int64_t read_varint64(VALUE self) { + int shift = 0; + int64_t result = 0; + while (true) { + int8_t b = read_byte_direct(self); + result = result | ((uint64_t)(b & 0x7f) << shift); + if ((b & 0x80) != 0x80) { + break; + } + shift += 7; + } + return result; +} + +static int16_t read_i16(VALUE self) { + return zig_zag_to_int((int32_t)read_varint64(self)); +} + +static VALUE get_protocol_exception(VALUE code, VALUE message) { + VALUE args[2]; + args[0] = code; + args[1] = message; + return rb_class_new_instance(2, (VALUE*)&args, protocol_exception_class); +} + +VALUE rb_thrift_compact_proto_read_message_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_compact_proto_read_struct_begin(VALUE self) { + rb_ary_push(rb_ivar_get(self, last_field_id), INT2FIX(0)); + return Qnil; +} + +VALUE rb_thrift_compact_proto_read_struct_end(VALUE self) { + rb_ary_pop(rb_ivar_get(self, last_field_id)); + return Qnil; +} + +VALUE rb_thrift_compact_proto_read_field_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_compact_proto_read_map_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_compact_proto_read_list_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_compact_proto_read_set_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_compact_proto_read_message_begin(VALUE self) { + int8_t protocol_id = read_byte_direct(self); + if (protocol_id != PROTOCOL_ID) { + char buf[100]; + int len = sprintf(buf, "Expected protocol id %d but got %d", PROTOCOL_ID, protocol_id); + buf[len] = 0; + rb_exc_raise(get_protocol_exception(INT2FIX(-1), rb_str_new2(buf))); + } + + int8_t version_and_type = read_byte_direct(self); + int8_t version = version_and_type & VERSION_MASK; + if (version != VERSION) { + char buf[100]; + int len = sprintf(buf, "Expected version id %d but got %d", version, VERSION); + buf[len] = 0; + rb_exc_raise(get_protocol_exception(INT2FIX(-1), rb_str_new2(buf))); + } + + int8_t type = (version_and_type >> TYPE_SHIFT_AMOUNT) & 0x03; + int32_t seqid = read_varint64(self); + VALUE messageName = rb_thrift_compact_proto_read_string(self); + return rb_ary_new3(3, messageName, INT2FIX(type), INT2NUM(seqid)); +} + +VALUE rb_thrift_compact_proto_read_field_begin(VALUE self) { + int8_t type = read_byte_direct(self); + // if it's a stop, then we can return immediately, as the struct is over. + if ((type & 0x0f) == TTYPE_STOP) { + return rb_ary_new3(3, Qnil, INT2FIX(0), INT2FIX(0)); + } else { + int field_id = 0; + + // mask off the 4 MSB of the type header. it could contain a field id delta. + uint8_t modifier = ((type & 0xf0) >> 4); + + if (modifier == 0) { + // not a delta. look ahead for the zigzag varint field id. + field_id = read_i16(self); + } else { + // has a delta. add the delta to the last read field id. + field_id = LAST_ID(self) + modifier; + } + + // if this happens to be a boolean field, the value is encoded in the type + if (is_bool_type(type)) { + // save the boolean value in a special instance variable. + rb_ivar_set(self, bool_value_id, (type & 0x0f) == CTYPE_BOOLEAN_TRUE ? Qtrue : Qfalse); + } + + // push the new field onto the field stack so we can keep the deltas going. + SET_LAST_ID(self, INT2FIX(field_id)); + return rb_ary_new3(3, Qnil, INT2FIX(get_ttype(type & 0x0f)), INT2FIX(field_id)); + } +} + +VALUE rb_thrift_compact_proto_read_map_begin(VALUE self) { + int32_t size = read_varint64(self); + uint8_t key_and_value_type = size == 0 ? 0 : read_byte_direct(self); + return rb_ary_new3(3, INT2FIX(get_ttype(key_and_value_type >> 4)), INT2FIX(get_ttype(key_and_value_type & 0xf)), INT2FIX(size)); +} + +VALUE rb_thrift_compact_proto_read_list_begin(VALUE self) { + uint8_t size_and_type = read_byte_direct(self); + int32_t size = (size_and_type >> 4) & 0x0f; + if (size == 15) { + size = read_varint64(self); + } + uint8_t type = get_ttype(size_and_type & 0x0f); + return rb_ary_new3(2, INT2FIX(type), INT2FIX(size)); +} + +VALUE rb_thrift_compact_proto_read_set_begin(VALUE self) { + return rb_thrift_compact_proto_read_list_begin(self); +} + +VALUE rb_thrift_compact_proto_read_bool(VALUE self) { + VALUE bool_value = rb_ivar_get(self, bool_value_id); + if (NIL_P(bool_value)) { + return read_byte_direct(self) == CTYPE_BOOLEAN_TRUE ? Qtrue : Qfalse; + } else { + rb_ivar_set(self, bool_value_id, Qnil); + return bool_value; + } +} + +VALUE rb_thrift_compact_proto_read_byte(VALUE self) { + return INT2FIX(read_byte_direct(self)); +} + +VALUE rb_thrift_compact_proto_read_i16(VALUE self) { + return INT2FIX(read_i16(self)); +} + +VALUE rb_thrift_compact_proto_read_i32(VALUE self) { + return INT2NUM(zig_zag_to_int(read_varint64(self))); +} + +VALUE rb_thrift_compact_proto_read_i64(VALUE self) { + return LL2NUM(zig_zag_to_ll(read_varint64(self))); +} + +VALUE rb_thrift_compact_proto_read_double(VALUE self) { + union { + double f; + int64_t l; + } transfer; + VALUE bytes = READ(self, 8); + uint32_t lo = ((uint8_t)(RSTRING_PTR(bytes)[0])) + | (((uint8_t)(RSTRING_PTR(bytes)[1])) << 8) + | (((uint8_t)(RSTRING_PTR(bytes)[2])) << 16) + | (((uint8_t)(RSTRING_PTR(bytes)[3])) << 24); + uint64_t hi = (((uint8_t)(RSTRING_PTR(bytes)[4]))) + | (((uint8_t)(RSTRING_PTR(bytes)[5])) << 8) + | (((uint8_t)(RSTRING_PTR(bytes)[6])) << 16) + | (((uint8_t)(RSTRING_PTR(bytes)[7])) << 24); + transfer.l = (hi << 32) | lo; + + return rb_float_new(transfer.f); +} + +VALUE rb_thrift_compact_proto_read_string(VALUE self) { + int64_t size = read_varint64(self); + return READ(self, size); +} + +static void Init_constants() { + thrift_compact_protocol_class = rb_const_get(thrift_module, rb_intern("CompactProtocol")); + + VERSION = rb_num2ll(rb_const_get(thrift_compact_protocol_class, rb_intern("VERSION"))); + VERSION_MASK = rb_num2ll(rb_const_get(thrift_compact_protocol_class, rb_intern("VERSION_MASK"))); + TYPE_MASK = rb_num2ll(rb_const_get(thrift_compact_protocol_class, rb_intern("TYPE_MASK"))); + TYPE_SHIFT_AMOUNT = FIX2INT(rb_const_get(thrift_compact_protocol_class, rb_intern("TYPE_SHIFT_AMOUNT"))); + PROTOCOL_ID = FIX2INT(rb_const_get(thrift_compact_protocol_class, rb_intern("PROTOCOL_ID"))); + + last_field_id = rb_intern("@last_field"); + boolean_field_id = rb_intern("@boolean_field"); + bool_value_id = rb_intern("@bool_value"); +} + +static void Init_rb_methods() { + rb_define_method(thrift_compact_protocol_class, "native?", rb_thrift_compact_proto_native_qmark, 0); + + rb_define_method(thrift_compact_protocol_class, "write_message_begin", rb_thrift_compact_proto_write_message_begin, 3); + rb_define_method(thrift_compact_protocol_class, "write_field_begin", rb_thrift_compact_proto_write_field_begin, 3); + rb_define_method(thrift_compact_protocol_class, "write_field_stop", rb_thrift_compact_proto_write_field_stop, 0); + rb_define_method(thrift_compact_protocol_class, "write_map_begin", rb_thrift_compact_proto_write_map_begin, 3); + rb_define_method(thrift_compact_protocol_class, "write_list_begin", rb_thrift_compact_proto_write_list_begin, 2); + rb_define_method(thrift_compact_protocol_class, "write_set_begin", rb_thrift_compact_proto_write_set_begin, 2); + rb_define_method(thrift_compact_protocol_class, "write_byte", rb_thrift_compact_proto_write_byte, 1); + rb_define_method(thrift_compact_protocol_class, "write_bool", rb_thrift_compact_proto_write_bool, 1); + rb_define_method(thrift_compact_protocol_class, "write_i16", rb_thrift_compact_proto_write_i16, 1); + rb_define_method(thrift_compact_protocol_class, "write_i32", rb_thrift_compact_proto_write_i32, 1); + rb_define_method(thrift_compact_protocol_class, "write_i64", rb_thrift_compact_proto_write_i64, 1); + rb_define_method(thrift_compact_protocol_class, "write_double", rb_thrift_compact_proto_write_double, 1); + rb_define_method(thrift_compact_protocol_class, "write_string", rb_thrift_compact_proto_write_string, 1); + + rb_define_method(thrift_compact_protocol_class, "write_message_end", rb_thrift_compact_proto_write_message_end, 0); + rb_define_method(thrift_compact_protocol_class, "write_struct_begin", rb_thrift_compact_proto_write_struct_begin, 1); + rb_define_method(thrift_compact_protocol_class, "write_struct_end", rb_thrift_compact_proto_write_struct_end, 0); + rb_define_method(thrift_compact_protocol_class, "write_field_end", rb_thrift_compact_proto_write_field_end, 0); + rb_define_method(thrift_compact_protocol_class, "write_map_end", rb_thrift_compact_proto_write_map_end, 0); + rb_define_method(thrift_compact_protocol_class, "write_list_end", rb_thrift_compact_proto_write_list_end, 0); + rb_define_method(thrift_compact_protocol_class, "write_set_end", rb_thrift_compact_proto_write_set_end, 0); + + + rb_define_method(thrift_compact_protocol_class, "read_message_begin", rb_thrift_compact_proto_read_message_begin, 0); + rb_define_method(thrift_compact_protocol_class, "read_field_begin", rb_thrift_compact_proto_read_field_begin, 0); + rb_define_method(thrift_compact_protocol_class, "read_map_begin", rb_thrift_compact_proto_read_map_begin, 0); + rb_define_method(thrift_compact_protocol_class, "read_list_begin", rb_thrift_compact_proto_read_list_begin, 0); + rb_define_method(thrift_compact_protocol_class, "read_set_begin", rb_thrift_compact_proto_read_set_begin, 0); + rb_define_method(thrift_compact_protocol_class, "read_byte", rb_thrift_compact_proto_read_byte, 0); + rb_define_method(thrift_compact_protocol_class, "read_bool", rb_thrift_compact_proto_read_bool, 0); + rb_define_method(thrift_compact_protocol_class, "read_i16", rb_thrift_compact_proto_read_i16, 0); + rb_define_method(thrift_compact_protocol_class, "read_i32", rb_thrift_compact_proto_read_i32, 0); + rb_define_method(thrift_compact_protocol_class, "read_i64", rb_thrift_compact_proto_read_i64, 0); + rb_define_method(thrift_compact_protocol_class, "read_double", rb_thrift_compact_proto_read_double, 0); + rb_define_method(thrift_compact_protocol_class, "read_string", rb_thrift_compact_proto_read_string, 0); + + rb_define_method(thrift_compact_protocol_class, "read_message_end", rb_thrift_compact_proto_read_message_end, 0); + rb_define_method(thrift_compact_protocol_class, "read_struct_begin", rb_thrift_compact_proto_read_struct_begin, 0); + rb_define_method(thrift_compact_protocol_class, "read_struct_end", rb_thrift_compact_proto_read_struct_end, 0); + rb_define_method(thrift_compact_protocol_class, "read_field_end", rb_thrift_compact_proto_read_field_end, 0); + rb_define_method(thrift_compact_protocol_class, "read_map_end", rb_thrift_compact_proto_read_map_end, 0); + rb_define_method(thrift_compact_protocol_class, "read_list_end", rb_thrift_compact_proto_read_list_end, 0); + rb_define_method(thrift_compact_protocol_class, "read_set_end", rb_thrift_compact_proto_read_set_end, 0); +} + +static void Init_npmt() { + native_proto_method_table *npmt; + npmt = ALLOC(native_proto_method_table); + + npmt->write_field_begin = rb_thrift_compact_proto_write_field_begin; + npmt->write_field_stop = rb_thrift_compact_proto_write_field_stop; + npmt->write_map_begin = rb_thrift_compact_proto_write_map_begin; + npmt->write_list_begin = rb_thrift_compact_proto_write_list_begin; + npmt->write_set_begin = rb_thrift_compact_proto_write_set_begin; + npmt->write_byte = rb_thrift_compact_proto_write_byte; + npmt->write_bool = rb_thrift_compact_proto_write_bool; + npmt->write_i16 = rb_thrift_compact_proto_write_i16; + npmt->write_i32 = rb_thrift_compact_proto_write_i32; + npmt->write_i64 = rb_thrift_compact_proto_write_i64; + npmt->write_double = rb_thrift_compact_proto_write_double; + npmt->write_string = rb_thrift_compact_proto_write_string; + npmt->write_message_end = rb_thrift_compact_proto_write_message_end; + npmt->write_struct_begin = rb_thrift_compact_proto_write_struct_begin; + npmt->write_struct_end = rb_thrift_compact_proto_write_struct_end; + npmt->write_field_end = rb_thrift_compact_proto_write_field_end; + npmt->write_map_end = rb_thrift_compact_proto_write_map_end; + npmt->write_list_end = rb_thrift_compact_proto_write_list_end; + npmt->write_set_end = rb_thrift_compact_proto_write_set_end; + + npmt->read_message_begin = rb_thrift_compact_proto_read_message_begin; + npmt->read_field_begin = rb_thrift_compact_proto_read_field_begin; + npmt->read_map_begin = rb_thrift_compact_proto_read_map_begin; + npmt->read_list_begin = rb_thrift_compact_proto_read_list_begin; + npmt->read_set_begin = rb_thrift_compact_proto_read_set_begin; + npmt->read_byte = rb_thrift_compact_proto_read_byte; + npmt->read_bool = rb_thrift_compact_proto_read_bool; + npmt->read_i16 = rb_thrift_compact_proto_read_i16; + npmt->read_i32 = rb_thrift_compact_proto_read_i32; + npmt->read_i64 = rb_thrift_compact_proto_read_i64; + npmt->read_double = rb_thrift_compact_proto_read_double; + npmt->read_string = rb_thrift_compact_proto_read_string; + npmt->read_message_end = rb_thrift_compact_proto_read_message_end; + npmt->read_struct_begin = rb_thrift_compact_proto_read_struct_begin; + npmt->read_struct_end = rb_thrift_compact_proto_read_struct_end; + npmt->read_field_end = rb_thrift_compact_proto_read_field_end; + npmt->read_map_end = rb_thrift_compact_proto_read_map_end; + npmt->read_list_end = rb_thrift_compact_proto_read_list_end; + npmt->read_set_end = rb_thrift_compact_proto_read_set_end; + + VALUE method_table_object = Data_Wrap_Struct(rb_cObject, 0, free, npmt); + rb_const_set(thrift_compact_protocol_class, rb_intern("@native_method_table"), method_table_object); +} + + + +void Init_compact_protocol() { + Init_constants(); + Init_rb_methods(); + Init_npmt(); +} diff --git a/lib/rb/ext/compact_protocol.h b/lib/rb/ext/compact_protocol.h new file mode 100644 index 00000000..163915e9 --- /dev/null +++ b/lib/rb/ext/compact_protocol.h @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +void Init_compact_protocol(); diff --git a/lib/rb/ext/constants.h b/lib/rb/ext/constants.h new file mode 100644 index 00000000..57df544b --- /dev/null +++ b/lib/rb/ext/constants.h @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +extern int TTYPE_STOP; +extern int TTYPE_BOOL; +extern int TTYPE_BYTE; +extern int TTYPE_I16; +extern int TTYPE_I32; +extern int TTYPE_I64; +extern int TTYPE_DOUBLE; +extern int TTYPE_STRING; +extern int TTYPE_MAP; +extern int TTYPE_SET; +extern int TTYPE_LIST; +extern int TTYPE_STRUCT; + +extern ID validate_method_id; +extern ID write_struct_begin_method_id; +extern ID write_struct_end_method_id; +extern ID write_field_begin_method_id; +extern ID write_field_end_method_id; +extern ID write_boolean_method_id; +extern ID write_byte_method_id; +extern ID write_i16_method_id; +extern ID write_i32_method_id; +extern ID write_i64_method_id; +extern ID write_double_method_id; +extern ID write_string_method_id; +extern ID write_map_begin_method_id; +extern ID write_map_end_method_id; +extern ID write_list_begin_method_id; +extern ID write_list_end_method_id; +extern ID write_set_begin_method_id; +extern ID write_set_end_method_id; +extern ID size_method_id; +extern ID read_bool_method_id; +extern ID read_byte_method_id; +extern ID read_i16_method_id; +extern ID read_i32_method_id; +extern ID read_i64_method_id; +extern ID read_string_method_id; +extern ID read_double_method_id; +extern ID read_map_begin_method_id; +extern ID read_map_end_method_id; +extern ID read_list_begin_method_id; +extern ID read_list_end_method_id; +extern ID read_set_begin_method_id; +extern ID read_set_end_method_id; +extern ID read_struct_begin_method_id; +extern ID read_struct_end_method_id; +extern ID read_field_begin_method_id; +extern ID read_field_end_method_id; +extern ID keys_method_id; +extern ID entries_method_id; +extern ID name_method_id; +extern ID sort_method_id; +extern ID write_field_stop_method_id; +extern ID skip_method_id; +extern ID write_method_id; +extern ID read_all_method_id; +extern ID native_qmark_method_id; + +extern ID fields_const_id; +extern ID transport_ivar_id; +extern ID strict_read_ivar_id; +extern ID strict_write_ivar_id; + +extern VALUE type_sym; +extern VALUE name_sym; +extern VALUE key_sym; +extern VALUE value_sym; +extern VALUE element_sym; +extern VALUE class_sym; + +extern VALUE rb_cSet; +extern VALUE thrift_module; +extern VALUE thrift_types_module; +extern VALUE class_thrift_protocol; +extern VALUE protocol_exception_class; diff --git a/lib/rb/ext/extconf.rb b/lib/rb/ext/extconf.rb new file mode 100644 index 00000000..07558b8a --- /dev/null +++ b/lib/rb/ext/extconf.rb @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'mkmf' + +$CFLAGS = "-g -O2 -Wall -Werror" + +have_func("strlcpy", "string.h") + +create_makefile 'thrift_native' diff --git a/lib/rb/ext/macros.h b/lib/rb/ext/macros.h new file mode 100644 index 00000000..265f6930 --- /dev/null +++ b/lib/rb/ext/macros.h @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#define GET_TRANSPORT(obj) rb_ivar_get(obj, transport_ivar_id) +#define GET_STRICT_READ(obj) rb_ivar_get(obj, strict_read_ivar_id) +#define GET_STRICT_WRITE(obj) rb_ivar_get(obj, strict_write_ivar_id) +#define WRITE(obj, data, length) rb_funcall(obj, write_method_id, 1, rb_str_new(data, length)) +#define CHECK_NIL(obj) if (NIL_P(obj)) { rb_raise(rb_eStandardError, "nil argument not allowed!");} +#define READ(obj, length) rb_funcall(GET_TRANSPORT(obj), read_all_method_id, 1, INT2FIX(length)) + +#ifndef RFLOAT_VALUE +# define RFLOAT_VALUE(v) RFLOAT(rb_Float(v))->value +#endif + +#ifndef RSTRING_LEN +# define RSTRING_LEN(v) RSTRING(rb_String(v))->len +#endif + +#ifndef RSTRING_PTR +# define RSTRING_PTR(v) RSTRING(rb_String(v))->ptr +#endif + +#ifndef RARRAY_LEN +# define RARRAY_LEN(v) RARRAY(rb_Array(v))->len +#endif diff --git a/lib/rb/ext/memory_buffer.c b/lib/rb/ext/memory_buffer.c new file mode 100644 index 00000000..624012d4 --- /dev/null +++ b/lib/rb/ext/memory_buffer.c @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include "macros.h" + +ID buf_ivar_id; +ID index_ivar_id; + +ID slice_method_id; + +int GARBAGE_BUFFER_SIZE; + +#define GET_BUF(self) rb_ivar_get(self, buf_ivar_id) + +VALUE rb_thrift_memory_buffer_write(VALUE self, VALUE str) { + VALUE buf = GET_BUF(self); + rb_str_buf_cat(buf, RSTRING_PTR(str), RSTRING_LEN(str)); + return Qnil; +} + +VALUE rb_thrift_memory_buffer_read(VALUE self, VALUE length_value) { + int length = FIX2INT(length_value); + + VALUE index_value = rb_ivar_get(self, index_ivar_id); + int index = FIX2INT(index_value); + + VALUE buf = GET_BUF(self); + VALUE data = rb_funcall(buf, slice_method_id, 2, index_value, length_value); + + index += length; + if (index > RSTRING_LEN(buf)) { + index = RSTRING_LEN(buf); + } + if (index >= GARBAGE_BUFFER_SIZE) { + rb_ivar_set(self, buf_ivar_id, rb_funcall(buf, slice_method_id, 2, INT2FIX(index), INT2FIX(RSTRING_LEN(buf) - 1))); + index = 0; + } + + rb_ivar_set(self, index_ivar_id, INT2FIX(index)); + return data; +} + +void Init_memory_buffer() { + VALUE thrift_memory_buffer_class = rb_const_get(thrift_module, rb_intern("MemoryBufferTransport")); + rb_define_method(thrift_memory_buffer_class, "write", rb_thrift_memory_buffer_write, 1); + rb_define_method(thrift_memory_buffer_class, "read", rb_thrift_memory_buffer_read, 1); + + buf_ivar_id = rb_intern("@buf"); + index_ivar_id = rb_intern("@index"); + + slice_method_id = rb_intern("slice"); + + GARBAGE_BUFFER_SIZE = FIX2INT(rb_const_get(thrift_memory_buffer_class, rb_intern("GARBAGE_BUFFER_SIZE"))); +} diff --git a/lib/rb/ext/memory_buffer.h b/lib/rb/ext/memory_buffer.h new file mode 100644 index 00000000..b277fa6f --- /dev/null +++ b/lib/rb/ext/memory_buffer.h @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +void Init_memory_buffer(); diff --git a/lib/rb/ext/protocol.c b/lib/rb/ext/protocol.c new file mode 100644 index 00000000..c1876541 --- /dev/null +++ b/lib/rb/ext/protocol.c @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +static VALUE skip(VALUE self, int ttype) { + if (ttype == TTYPE_STOP) { + return Qnil; + } else if (ttype == TTYPE_BOOL) { + rb_funcall(self, read_bool_method_id, 0); + } else if (ttype == TTYPE_BYTE) { + rb_funcall(self, read_byte_method_id, 0); + } else if (ttype == TTYPE_I16) { + rb_funcall(self, read_i16_method_id, 0); + } else if (ttype == TTYPE_I32) { + rb_funcall(self, read_i32_method_id, 0); + } else if (ttype == TTYPE_I64) { + rb_funcall(self, read_i64_method_id, 0); + } else if (ttype == TTYPE_DOUBLE) { + rb_funcall(self, read_double_method_id, 0); + } else if (ttype == TTYPE_STRING) { + rb_funcall(self, read_string_method_id, 0); + } else if (ttype == TTYPE_STRUCT) { + rb_funcall(self, read_struct_begin_method_id, 0); + while (true) { + VALUE field_header = rb_funcall(self, read_field_begin_method_id, 0); + if (NIL_P(field_header) || FIX2INT(rb_ary_entry(field_header, 1)) == TTYPE_STOP ) { + break; + } + skip(self, FIX2INT(rb_ary_entry(field_header, 1))); + rb_funcall(self, read_field_end_method_id, 0); + } + rb_funcall(self, read_struct_end_method_id, 0); + } else if (ttype == TTYPE_MAP) { + int i; + VALUE map_header = rb_funcall(self, read_map_begin_method_id, 0); + int ktype = FIX2INT(rb_ary_entry(map_header, 0)); + int vtype = FIX2INT(rb_ary_entry(map_header, 1)); + int size = FIX2INT(rb_ary_entry(map_header, 2)); + + for (i = 0; i < size; i++) { + skip(self, ktype); + skip(self, vtype); + } + rb_funcall(self, read_map_end_method_id, 0); + } else if (ttype == TTYPE_LIST || ttype == TTYPE_SET) { + int i; + VALUE collection_header = rb_funcall(self, ttype == TTYPE_LIST ? read_list_begin_method_id : read_set_begin_method_id, 0); + int etype = FIX2INT(rb_ary_entry(collection_header, 0)); + int size = FIX2INT(rb_ary_entry(collection_header, 1)); + for (i = 0; i < size; i++) { + skip(self, etype); + } + rb_funcall(self, ttype == TTYPE_LIST ? read_list_end_method_id : read_set_end_method_id, 0); + } else { + rb_raise(rb_eNotImpError, "don't know how to skip type %d", ttype); + } + + return Qnil; +} + +VALUE rb_thrift_protocol_native_qmark(VALUE self) { + return Qfalse; +} + +VALUE rb_thrift_protocol_skip(VALUE protocol, VALUE ttype) { + return skip(protocol, FIX2INT(ttype)); +} + +VALUE rb_thrift_write_message_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_write_struct_begin(VALUE self, VALUE name) { + return Qnil; +} + +VALUE rb_thrift_write_struct_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_write_field_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_write_map_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_write_list_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_write_set_end(VALUE self) { + return Qnil; +} + +VALUE rb_thrift_read_message_end(VALUE self) { + return Qnil; +} + +VALUE rb_thift_read_struct_begin(VALUE self) { + return Qnil; +} + +VALUE rb_thift_read_struct_end(VALUE self) { + return Qnil; +} + +VALUE rb_thift_read_field_end(VALUE self) { + return Qnil; +} + +VALUE rb_thift_read_map_end(VALUE self) { + return Qnil; +} + +VALUE rb_thift_read_list_end(VALUE self) { + return Qnil; +} + +VALUE rb_thift_read_set_end(VALUE self) { + return Qnil; +} + +void Init_protocol() { + VALUE c_protocol = rb_const_get(thrift_module, rb_intern("BaseProtocol")); + + rb_define_method(c_protocol, "skip", rb_thrift_protocol_skip, 1); + rb_define_method(c_protocol, "write_message_end", rb_thrift_write_message_end, 0); + rb_define_method(c_protocol, "write_struct_begin", rb_thrift_write_struct_begin, 1); + rb_define_method(c_protocol, "write_struct_end", rb_thrift_write_struct_end, 0); + rb_define_method(c_protocol, "write_field_end", rb_thrift_write_field_end, 0); + rb_define_method(c_protocol, "write_map_end", rb_thrift_write_map_end, 0); + rb_define_method(c_protocol, "write_list_end", rb_thrift_write_list_end, 0); + rb_define_method(c_protocol, "write_set_end", rb_thrift_write_set_end, 0); + rb_define_method(c_protocol, "read_message_end", rb_thrift_read_message_end, 0); + rb_define_method(c_protocol, "read_struct_begin", rb_thift_read_struct_begin, 0); + rb_define_method(c_protocol, "read_struct_end", rb_thift_read_struct_end, 0); + rb_define_method(c_protocol, "read_field_end", rb_thift_read_field_end, 0); + rb_define_method(c_protocol, "read_map_end", rb_thift_read_map_end, 0); + rb_define_method(c_protocol, "read_list_end", rb_thift_read_list_end, 0); + rb_define_method(c_protocol, "read_set_end", rb_thift_read_set_end, 0); + rb_define_method(c_protocol, "native?", rb_thrift_protocol_native_qmark, 0); + + // native_proto_method_table *npmt; + // npmt = ALLOC(native_proto_method_table); + // npmt->write_message_end = rb_thrift_write_message_end; + // npmt->write_struct_begin = rb_thrift_write_struct_begin; + // npmt->write_struct_end = rb_thrift_write_struct_end; + // npmt->write_field_end = rb_thrift_write_field_end; + // npmt->write_map_end = rb_thrift_write_map_end; + // npmt->write_list_end = rb_thrift_write_list_end; + // npmt->write_set_end = rb_thrift_write_set_end; + // npmt->read_message_end = rb_thrift_read_message_end; + // npmt->read_struct_begin = rb_thift_read_struct_begin; + // npmt->read_struct_end = rb_thift_read_struct_end; + // npmt->read_field_end = rb_thift_read_field_end; + // npmt->read_map_end = rb_thift_read_map_end; + // npmt->read_list_end = rb_thift_read_list_end; + // npmt->read_set_end = rb_thift_read_set_end; + // + // VALUE method_table_object = Data_Wrap_Struct(rb_cObject, 0, free, npmt); + // rb_const_set(c_protocol, rb_intern("@native_method_table"), method_table_object); +} diff --git a/lib/rb/ext/protocol.h b/lib/rb/ext/protocol.h new file mode 100644 index 00000000..53695303 --- /dev/null +++ b/lib/rb/ext/protocol.h @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +void Init_protocol(); diff --git a/lib/rb/ext/struct.c b/lib/rb/ext/struct.c new file mode 100644 index 00000000..fee285e9 --- /dev/null +++ b/lib/rb/ext/struct.c @@ -0,0 +1,605 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include "macros.h" + +#ifndef HAVE_STRLCPY + +static +size_t +strlcpy (char *dst, const char *src, size_t dst_sz) +{ + size_t n; + + for (n = 0; n < dst_sz; n++) { + if ((*dst++ = *src++) == '\0') + break; + } + + if (n < dst_sz) + return n; + if (n > 0) + *(dst - 1) = '\0'; + return n + strlen (src); +} + +#endif + +static native_proto_method_table *mt; +static native_proto_method_table *default_mt; +static VALUE last_proto_class = Qnil; + +#define IS_CONTAINER(ttype) ((ttype) == TTYPE_MAP || (ttype) == TTYPE_LIST || (ttype) == TTYPE_SET) +#define STRUCT_FIELDS(obj) rb_const_get(CLASS_OF(obj), fields_const_id) + +static void set_native_proto_function_pointers(VALUE protocol) { + VALUE method_table_object = rb_const_get(CLASS_OF(protocol), rb_intern("@native_method_table")); + // TODO: check nil? + Data_Get_Struct(method_table_object, native_proto_method_table, mt); +} + +static void check_native_proto_method_table(VALUE protocol) { + VALUE protoclass = CLASS_OF(protocol); + if (protoclass != last_proto_class) { + last_proto_class = protoclass; + if (rb_funcall(protocol, native_qmark_method_id, 0) == Qtrue) { + set_native_proto_function_pointers(protocol); + } else { + mt = default_mt; + } + } +} + +//------------------------------------------- +// Writing section +//------------------------------------------- + +// default fn pointers for protocol stuff here + +VALUE default_write_bool(VALUE protocol, VALUE value) { + rb_funcall(protocol, write_boolean_method_id, 1, value); + return Qnil; +} + +VALUE default_write_byte(VALUE protocol, VALUE value) { + rb_funcall(protocol, write_byte_method_id, 1, value); + return Qnil; +} + +VALUE default_write_i16(VALUE protocol, VALUE value) { + rb_funcall(protocol, write_i16_method_id, 1, value); + return Qnil; +} + +VALUE default_write_i32(VALUE protocol, VALUE value) { + rb_funcall(protocol, write_i32_method_id, 1, value); + return Qnil; +} + +VALUE default_write_i64(VALUE protocol, VALUE value) { + rb_funcall(protocol, write_i64_method_id, 1, value); + return Qnil; +} + +VALUE default_write_double(VALUE protocol, VALUE value) { + rb_funcall(protocol, write_double_method_id, 1, value); + return Qnil; +} + +VALUE default_write_string(VALUE protocol, VALUE value) { + rb_funcall(protocol, write_string_method_id, 1, value); + return Qnil; +} + +VALUE default_write_list_begin(VALUE protocol, VALUE etype, VALUE length) { + rb_funcall(protocol, write_list_begin_method_id, 2, etype, length); + return Qnil; +} + +VALUE default_write_list_end(VALUE protocol) { + rb_funcall(protocol, write_list_end_method_id, 0); + return Qnil; +} + +VALUE default_write_set_begin(VALUE protocol, VALUE etype, VALUE length) { + rb_funcall(protocol, write_set_begin_method_id, 2, etype, length); + return Qnil; +} + +VALUE default_write_set_end(VALUE protocol) { + rb_funcall(protocol, write_set_end_method_id, 0); + return Qnil; +} + +VALUE default_write_map_begin(VALUE protocol, VALUE ktype, VALUE vtype, VALUE length) { + rb_funcall(protocol, write_map_begin_method_id, 3, ktype, vtype, length); + return Qnil; +} + +VALUE default_write_map_end(VALUE protocol) { + rb_funcall(protocol, write_map_end_method_id, 0); + return Qnil; +} + +VALUE default_write_struct_begin(VALUE protocol, VALUE struct_name) { + rb_funcall(protocol, write_struct_begin_method_id, 1, struct_name); + return Qnil; +} + +VALUE default_write_struct_end(VALUE protocol) { + rb_funcall(protocol, write_struct_end_method_id, 0); + return Qnil; +} + +VALUE default_write_field_begin(VALUE protocol, VALUE name, VALUE type, VALUE id) { + rb_funcall(protocol, write_field_begin_method_id, 3, name, type, id); + return Qnil; +} + +VALUE default_write_field_end(VALUE protocol) { + rb_funcall(protocol, write_field_end_method_id, 0); + return Qnil; +} + +VALUE default_write_field_stop(VALUE protocol) { + rb_funcall(protocol, write_field_stop_method_id, 0); + return Qnil; +} + +VALUE default_read_field_begin(VALUE protocol) { + return rb_funcall(protocol, read_field_begin_method_id, 0); +} + +VALUE default_read_field_end(VALUE protocol) { + return rb_funcall(protocol, read_field_end_method_id, 0); +} + +VALUE default_read_map_begin(VALUE protocol) { + return rb_funcall(protocol, read_map_begin_method_id, 0); +} + +VALUE default_read_map_end(VALUE protocol) { + return rb_funcall(protocol, read_map_end_method_id, 0); +} + +VALUE default_read_list_begin(VALUE protocol) { + return rb_funcall(protocol, read_list_begin_method_id, 0); +} + +VALUE default_read_list_end(VALUE protocol) { + return rb_funcall(protocol, read_list_end_method_id, 0); +} + +VALUE default_read_set_begin(VALUE protocol) { + return rb_funcall(protocol, read_set_begin_method_id, 0); +} + +VALUE default_read_set_end(VALUE protocol) { + return rb_funcall(protocol, read_set_end_method_id, 0); +} + +VALUE default_read_byte(VALUE protocol) { + return rb_funcall(protocol, read_byte_method_id, 0); +} + +VALUE default_read_bool(VALUE protocol) { + return rb_funcall(protocol, read_bool_method_id, 0); +} + +VALUE default_read_i16(VALUE protocol) { + return rb_funcall(protocol, read_i16_method_id, 0); +} + +VALUE default_read_i32(VALUE protocol) { + return rb_funcall(protocol, read_i32_method_id, 0); +} + +VALUE default_read_i64(VALUE protocol) { + return rb_funcall(protocol, read_i64_method_id, 0); +} + +VALUE default_read_double(VALUE protocol) { + return rb_funcall(protocol, read_double_method_id, 0); +} + +VALUE default_read_string(VALUE protocol) { + return rb_funcall(protocol, read_string_method_id, 0); +} + +VALUE default_read_struct_begin(VALUE protocol) { + return rb_funcall(protocol, read_struct_begin_method_id, 0); +} + +VALUE default_read_struct_end(VALUE protocol) { + return rb_funcall(protocol, read_struct_end_method_id, 0); +} + +static void set_default_proto_function_pointers() { + default_mt = ALLOC(native_proto_method_table); + + default_mt->write_field_begin = default_write_field_begin; + default_mt->write_field_stop = default_write_field_stop; + default_mt->write_map_begin = default_write_map_begin; + default_mt->write_map_end = default_write_map_end; + default_mt->write_list_begin = default_write_list_begin; + default_mt->write_list_end = default_write_list_end; + default_mt->write_set_begin = default_write_set_begin; + default_mt->write_set_end = default_write_set_end; + default_mt->write_byte = default_write_byte; + default_mt->write_bool = default_write_bool; + default_mt->write_i16 = default_write_i16; + default_mt->write_i32 = default_write_i32; + default_mt->write_i64 = default_write_i64; + default_mt->write_double = default_write_double; + default_mt->write_string = default_write_string; + default_mt->write_struct_begin = default_write_struct_begin; + default_mt->write_struct_end = default_write_struct_end; + default_mt->write_field_end = default_write_field_end; + + default_mt->read_struct_begin = default_read_struct_begin; + default_mt->read_struct_end = default_read_struct_end; + default_mt->read_field_begin = default_read_field_begin; + default_mt->read_field_end = default_read_field_end; + default_mt->read_map_begin = default_read_map_begin; + default_mt->read_map_end = default_read_map_end; + default_mt->read_list_begin = default_read_list_begin; + default_mt->read_list_end = default_read_list_end; + default_mt->read_set_begin = default_read_set_begin; + default_mt->read_set_end = default_read_set_end; + default_mt->read_byte = default_read_byte; + default_mt->read_bool = default_read_bool; + default_mt->read_i16 = default_read_i16; + default_mt->read_i32 = default_read_i32; + default_mt->read_i64 = default_read_i64; + default_mt->read_double = default_read_double; + default_mt->read_string = default_read_string; +} + +// end default protocol methods + + +static VALUE rb_thrift_struct_write(VALUE self, VALUE protocol); +static void write_anything(int ttype, VALUE value, VALUE protocol, VALUE field_info); + +VALUE get_field_value(VALUE obj, VALUE field_name) { + char name_buf[RSTRING_LEN(field_name) + 1]; + + name_buf[0] = '@'; + strlcpy(&name_buf[1], RSTRING_PTR(field_name), sizeof(name_buf)); + + VALUE value = rb_ivar_get(obj, rb_intern(name_buf)); + + return value; +} + +static void write_container(int ttype, VALUE field_info, VALUE value, VALUE protocol) { + int sz, i; + + if (ttype == TTYPE_MAP) { + VALUE keys; + VALUE key; + VALUE val; + + Check_Type(value, T_HASH); + + VALUE key_info = rb_hash_aref(field_info, key_sym); + VALUE keytype_value = rb_hash_aref(key_info, type_sym); + int keytype = FIX2INT(keytype_value); + + VALUE value_info = rb_hash_aref(field_info, value_sym); + VALUE valuetype_value = rb_hash_aref(value_info, type_sym); + int valuetype = FIX2INT(valuetype_value); + + keys = rb_funcall(value, keys_method_id, 0); + + sz = RARRAY_LEN(keys); + + mt->write_map_begin(protocol, keytype_value, valuetype_value, INT2FIX(sz)); + + for (i = 0; i < sz; i++) { + key = rb_ary_entry(keys, i); + val = rb_hash_aref(value, key); + + if (IS_CONTAINER(keytype)) { + write_container(keytype, key_info, key, protocol); + } else { + write_anything(keytype, key, protocol, key_info); + } + + if (IS_CONTAINER(valuetype)) { + write_container(valuetype, value_info, val, protocol); + } else { + write_anything(valuetype, val, protocol, value_info); + } + } + + mt->write_map_end(protocol); + } else if (ttype == TTYPE_LIST) { + Check_Type(value, T_ARRAY); + + sz = RARRAY_LEN(value); + + VALUE element_type_info = rb_hash_aref(field_info, element_sym); + VALUE element_type_value = rb_hash_aref(element_type_info, type_sym); + int element_type = FIX2INT(element_type_value); + + mt->write_list_begin(protocol, element_type_value, INT2FIX(sz)); + for (i = 0; i < sz; ++i) { + VALUE val = rb_ary_entry(value, i); + if (IS_CONTAINER(element_type)) { + write_container(element_type, element_type_info, val, protocol); + } else { + write_anything(element_type, val, protocol, element_type_info); + } + } + mt->write_list_end(protocol); + } else if (ttype == TTYPE_SET) { + VALUE items; + + if (TYPE(value) == T_ARRAY) { + items = value; + } else { + if (rb_cSet == CLASS_OF(value)) { + items = rb_funcall(value, entries_method_id, 0); + } else { + Check_Type(value, T_HASH); + items = rb_funcall(value, keys_method_id, 0); + } + } + + sz = RARRAY_LEN(items); + + VALUE element_type_info = rb_hash_aref(field_info, element_sym); + VALUE element_type_value = rb_hash_aref(element_type_info, type_sym); + int element_type = FIX2INT(element_type_value); + + mt->write_set_begin(protocol, element_type_value, INT2FIX(sz)); + + for (i = 0; i < sz; i++) { + VALUE val = rb_ary_entry(items, i); + if (IS_CONTAINER(element_type)) { + write_container(element_type, element_type_info, val, protocol); + } else { + write_anything(element_type, val, protocol, element_type_info); + } + } + + mt->write_set_end(protocol); + } else { + rb_raise(rb_eNotImpError, "can't write container of type: %d", ttype); + } +} + +static void write_anything(int ttype, VALUE value, VALUE protocol, VALUE field_info) { + if (ttype == TTYPE_BOOL) { + mt->write_bool(protocol, value); + } else if (ttype == TTYPE_BYTE) { + mt->write_byte(protocol, value); + } else if (ttype == TTYPE_I16) { + mt->write_i16(protocol, value); + } else if (ttype == TTYPE_I32) { + mt->write_i32(protocol, value); + } else if (ttype == TTYPE_I64) { + mt->write_i64(protocol, value); + } else if (ttype == TTYPE_DOUBLE) { + mt->write_double(protocol, value); + } else if (ttype == TTYPE_STRING) { + mt->write_string(protocol, value); + } else if (IS_CONTAINER(ttype)) { + write_container(ttype, field_info, value, protocol); + } else if (ttype == TTYPE_STRUCT) { + rb_thrift_struct_write(value, protocol); + } else { + rb_raise(rb_eNotImpError, "Unknown type for binary_encoding: %d", ttype); + } +} + +static VALUE rb_thrift_struct_write(VALUE self, VALUE protocol) { + // call validate + rb_funcall(self, validate_method_id, 0); + + check_native_proto_method_table(protocol); + + // write struct begin + mt->write_struct_begin(protocol, rb_class_name(CLASS_OF(self))); + + // iterate through all the fields here + VALUE struct_fields = STRUCT_FIELDS(self); + VALUE struct_field_ids_unordered = rb_funcall(struct_fields, keys_method_id, 0); + VALUE struct_field_ids_ordered = rb_funcall(struct_field_ids_unordered, sort_method_id, 0); + + int i = 0; + for (i=0; i < RARRAY_LEN(struct_field_ids_ordered); i++) { + VALUE field_id = rb_ary_entry(struct_field_ids_ordered, i); + VALUE field_info = rb_hash_aref(struct_fields, field_id); + + VALUE ttype_value = rb_hash_aref(field_info, type_sym); + int ttype = FIX2INT(ttype_value); + VALUE field_name = rb_hash_aref(field_info, name_sym); + VALUE field_value = get_field_value(self, field_name); + + if (!NIL_P(field_value)) { + mt->write_field_begin(protocol, field_name, ttype_value, field_id); + + write_anything(ttype, field_value, protocol, field_info); + + mt->write_field_end(protocol); + } + } + + mt->write_field_stop(protocol); + + // write struct end + mt->write_struct_end(protocol); + + return Qnil; +} + +//------------------------------------------- +// Reading section +//------------------------------------------- + +static VALUE rb_thrift_struct_read(VALUE self, VALUE protocol); + +static void set_field_value(VALUE obj, VALUE field_name, VALUE value) { + char name_buf[RSTRING_LEN(field_name) + 1]; + + name_buf[0] = '@'; + strlcpy(&name_buf[1], RSTRING_PTR(field_name), sizeof(name_buf)); + + rb_ivar_set(obj, rb_intern(name_buf), value); +} + +static VALUE read_anything(VALUE protocol, int ttype, VALUE field_info) { + VALUE result = Qnil; + + if (ttype == TTYPE_BOOL) { + result = mt->read_bool(protocol); + } else if (ttype == TTYPE_BYTE) { + result = mt->read_byte(protocol); + } else if (ttype == TTYPE_I16) { + result = mt->read_i16(protocol); + } else if (ttype == TTYPE_I32) { + result = mt->read_i32(protocol); + } else if (ttype == TTYPE_I64) { + result = mt->read_i64(protocol); + } else if (ttype == TTYPE_STRING) { + result = mt->read_string(protocol); + } else if (ttype == TTYPE_DOUBLE) { + result = mt->read_double(protocol); + } else if (ttype == TTYPE_STRUCT) { + VALUE klass = rb_hash_aref(field_info, class_sym); + result = rb_class_new_instance(0, NULL, klass); + rb_thrift_struct_read(result, protocol); + } else if (ttype == TTYPE_MAP) { + int i; + + VALUE map_header = mt->read_map_begin(protocol); + int key_ttype = FIX2INT(rb_ary_entry(map_header, 0)); + int value_ttype = FIX2INT(rb_ary_entry(map_header, 1)); + int num_entries = FIX2INT(rb_ary_entry(map_header, 2)); + + VALUE key_info = rb_hash_aref(field_info, key_sym); + VALUE value_info = rb_hash_aref(field_info, value_sym); + + result = rb_hash_new(); + + for (i = 0; i < num_entries; ++i) { + VALUE key, val; + + key = read_anything(protocol, key_ttype, key_info); + val = read_anything(protocol, value_ttype, value_info); + + rb_hash_aset(result, key, val); + } + + mt->read_map_end(protocol); + } else if (ttype == TTYPE_LIST) { + int i; + + VALUE list_header = mt->read_list_begin(protocol); + int element_ttype = FIX2INT(rb_ary_entry(list_header, 0)); + int num_elements = FIX2INT(rb_ary_entry(list_header, 1)); + result = rb_ary_new2(num_elements); + + for (i = 0; i < num_elements; ++i) { + rb_ary_push(result, read_anything(protocol, element_ttype, rb_hash_aref(field_info, element_sym))); + } + + + mt->read_list_end(protocol); + } else if (ttype == TTYPE_SET) { + VALUE items; + int i; + + VALUE set_header = mt->read_set_begin(protocol); + int element_ttype = FIX2INT(rb_ary_entry(set_header, 0)); + int num_elements = FIX2INT(rb_ary_entry(set_header, 1)); + items = rb_ary_new2(num_elements); + + for (i = 0; i < num_elements; ++i) { + rb_ary_push(items, read_anything(protocol, element_ttype, rb_hash_aref(field_info, element_sym))); + } + + + mt->read_set_end(protocol); + + result = rb_class_new_instance(1, &items, rb_cSet); + } else { + rb_raise(rb_eNotImpError, "read_anything not implemented for type %d!", ttype); + } + + return result; +} + +static VALUE rb_thrift_struct_read(VALUE self, VALUE protocol) { + check_native_proto_method_table(protocol); + + // read struct begin + mt->read_struct_begin(protocol); + + VALUE struct_fields = STRUCT_FIELDS(self); + + // read each field + while (true) { + VALUE field_header = mt->read_field_begin(protocol); + VALUE field_type_value = rb_ary_entry(field_header, 1); + int field_type = FIX2INT(field_type_value); + + if (field_type == TTYPE_STOP) { + break; + } + + // make sure we got a type we expected + VALUE field_info = rb_hash_aref(struct_fields, rb_ary_entry(field_header, 2)); + + if (!NIL_P(field_info)) { + int specified_type = FIX2INT(rb_hash_aref(field_info, type_sym)); + if (field_type == specified_type) { + // read the value + VALUE name = rb_hash_aref(field_info, name_sym); + set_field_value(self, name, read_anything(protocol, field_type, field_info)); + } else { + rb_funcall(protocol, skip_method_id, 1, field_type_value); + } + } else { + rb_funcall(protocol, skip_method_id, 1, field_type_value); + } + + // read field end + mt->read_field_end(protocol); + } + + // read struct end + mt->read_struct_end(protocol); + + return Qnil; +} + +void Init_struct() { + VALUE struct_module = rb_const_get(thrift_module, rb_intern("Struct")); + + rb_define_method(struct_module, "write", rb_thrift_struct_write, 1); + rb_define_method(struct_module, "read", rb_thrift_struct_read, 1); + + set_default_proto_function_pointers(); +} + diff --git a/lib/rb/ext/struct.h b/lib/rb/ext/struct.h new file mode 100644 index 00000000..37b1b35b --- /dev/null +++ b/lib/rb/ext/struct.h @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +typedef struct native_proto_method_table { + VALUE (*write_bool)(VALUE, VALUE); + VALUE (*write_byte)(VALUE, VALUE); + VALUE (*write_i16)(VALUE, VALUE); + VALUE (*write_i32)(VALUE, VALUE); + VALUE (*write_i64)(VALUE, VALUE); + VALUE (*write_double)(VALUE, VALUE); + VALUE (*write_string)(VALUE, VALUE); + VALUE (*write_list_begin)(VALUE, VALUE, VALUE); + VALUE (*write_list_end)(VALUE); + VALUE (*write_set_begin)(VALUE, VALUE, VALUE); + VALUE (*write_set_end)(VALUE); + VALUE (*write_map_begin)(VALUE, VALUE, VALUE, VALUE); + VALUE (*write_map_end)(VALUE); + VALUE (*write_struct_begin)(VALUE, VALUE); + VALUE (*write_struct_end)(VALUE); + VALUE (*write_field_begin)(VALUE, VALUE, VALUE, VALUE); + VALUE (*write_field_end)(VALUE); + VALUE (*write_field_stop)(VALUE); + VALUE (*write_message_begin)(VALUE, VALUE, VALUE, VALUE); + VALUE (*write_message_end)(VALUE); + + VALUE (*read_message_begin)(VALUE); + VALUE (*read_message_end)(VALUE); + VALUE (*read_field_begin)(VALUE); + VALUE (*read_field_end)(VALUE); + VALUE (*read_map_begin)(VALUE); + VALUE (*read_map_end)(VALUE); + VALUE (*read_list_begin)(VALUE); + VALUE (*read_list_end)(VALUE); + VALUE (*read_set_begin)(VALUE); + VALUE (*read_set_end)(VALUE); + VALUE (*read_byte)(VALUE); + VALUE (*read_bool)(VALUE); + VALUE (*read_i16)(VALUE); + VALUE (*read_i32)(VALUE); + VALUE (*read_i64)(VALUE); + VALUE (*read_double)(VALUE); + VALUE (*read_string)(VALUE); + VALUE (*read_struct_begin)(VALUE); + VALUE (*read_struct_end)(VALUE); + +} native_proto_method_table; + +void Init_struct(); diff --git a/lib/rb/ext/thrift_native.c b/lib/rb/ext/thrift_native.c new file mode 100644 index 00000000..effa202c --- /dev/null +++ b/lib/rb/ext/thrift_native.c @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +// cached classes/modules +VALUE rb_cSet; +VALUE thrift_module; +VALUE thrift_types_module; + +// TType constants +int TTYPE_STOP; +int TTYPE_BOOL; +int TTYPE_BYTE; +int TTYPE_I16; +int TTYPE_I32; +int TTYPE_I64; +int TTYPE_DOUBLE; +int TTYPE_STRING; +int TTYPE_MAP; +int TTYPE_SET; +int TTYPE_LIST; +int TTYPE_STRUCT; + +// method ids +ID validate_method_id; +ID write_struct_begin_method_id; +ID write_struct_end_method_id; +ID write_field_begin_method_id; +ID write_field_end_method_id; +ID write_boolean_method_id; +ID write_byte_method_id; +ID write_i16_method_id; +ID write_i32_method_id; +ID write_i64_method_id; +ID write_double_method_id; +ID write_string_method_id; +ID write_map_begin_method_id; +ID write_map_end_method_id; +ID write_list_begin_method_id; +ID write_list_end_method_id; +ID write_set_begin_method_id; +ID write_set_end_method_id; +ID size_method_id; +ID read_bool_method_id; +ID read_byte_method_id; +ID read_i16_method_id; +ID read_i32_method_id; +ID read_i64_method_id; +ID read_string_method_id; +ID read_double_method_id; +ID read_map_begin_method_id; +ID read_map_end_method_id; +ID read_list_begin_method_id; +ID read_list_end_method_id; +ID read_set_begin_method_id; +ID read_set_end_method_id; +ID read_struct_begin_method_id; +ID read_struct_end_method_id; +ID read_field_begin_method_id; +ID read_field_end_method_id; +ID keys_method_id; +ID entries_method_id; +ID name_method_id; +ID sort_method_id; +ID write_field_stop_method_id; +ID skip_method_id; +ID write_method_id; +ID read_all_method_id; +ID native_qmark_method_id; + +// constant ids +ID fields_const_id; +ID transport_ivar_id; +ID strict_read_ivar_id; +ID strict_write_ivar_id; + +// cached symbols +VALUE type_sym; +VALUE name_sym; +VALUE key_sym; +VALUE value_sym; +VALUE element_sym; +VALUE class_sym; +VALUE protocol_exception_class; + +void Init_thrift_native() { + // cached classes + thrift_module = rb_const_get(rb_cObject, rb_intern("Thrift")); + thrift_types_module = rb_const_get(thrift_module, rb_intern("Types")); + rb_cSet = rb_const_get(rb_cObject, rb_intern("Set")); + protocol_exception_class = rb_const_get(thrift_module, rb_intern("ProtocolException")); + + // Init ttype constants + TTYPE_BOOL = FIX2INT(rb_const_get(thrift_types_module, rb_intern("BOOL"))); + TTYPE_BYTE = FIX2INT(rb_const_get(thrift_types_module, rb_intern("BYTE"))); + TTYPE_I16 = FIX2INT(rb_const_get(thrift_types_module, rb_intern("I16"))); + TTYPE_I32 = FIX2INT(rb_const_get(thrift_types_module, rb_intern("I32"))); + TTYPE_I64 = FIX2INT(rb_const_get(thrift_types_module, rb_intern("I64"))); + TTYPE_DOUBLE = FIX2INT(rb_const_get(thrift_types_module, rb_intern("DOUBLE"))); + TTYPE_STRING = FIX2INT(rb_const_get(thrift_types_module, rb_intern("STRING"))); + TTYPE_MAP = FIX2INT(rb_const_get(thrift_types_module, rb_intern("MAP"))); + TTYPE_SET = FIX2INT(rb_const_get(thrift_types_module, rb_intern("SET"))); + TTYPE_LIST = FIX2INT(rb_const_get(thrift_types_module, rb_intern("LIST"))); + TTYPE_STRUCT = FIX2INT(rb_const_get(thrift_types_module, rb_intern("STRUCT"))); + + // method ids + validate_method_id = rb_intern("validate"); + write_struct_begin_method_id = rb_intern("write_struct_begin"); + write_struct_end_method_id = rb_intern("write_struct_end"); + write_field_begin_method_id = rb_intern("write_field_begin"); + write_field_end_method_id = rb_intern("write_field_end"); + write_boolean_method_id = rb_intern("write_bool"); + write_byte_method_id = rb_intern("write_byte"); + write_i16_method_id = rb_intern("write_i16"); + write_i32_method_id = rb_intern("write_i32"); + write_i64_method_id = rb_intern("write_i64"); + write_double_method_id = rb_intern("write_double"); + write_string_method_id = rb_intern("write_string"); + write_map_begin_method_id = rb_intern("write_map_begin"); + write_map_end_method_id = rb_intern("write_map_end"); + write_list_begin_method_id = rb_intern("write_list_begin"); + write_list_end_method_id = rb_intern("write_list_end"); + write_set_begin_method_id = rb_intern("write_set_begin"); + write_set_end_method_id = rb_intern("write_set_end"); + size_method_id = rb_intern("size"); + read_bool_method_id = rb_intern("read_bool"); + read_byte_method_id = rb_intern("read_byte"); + read_i16_method_id = rb_intern("read_i16"); + read_i32_method_id = rb_intern("read_i32"); + read_i64_method_id = rb_intern("read_i64"); + read_string_method_id = rb_intern("read_string"); + read_double_method_id = rb_intern("read_double"); + read_map_begin_method_id = rb_intern("read_map_begin"); + read_map_end_method_id = rb_intern("read_map_end"); + read_list_begin_method_id = rb_intern("read_list_begin"); + read_list_end_method_id = rb_intern("read_list_end"); + read_set_begin_method_id = rb_intern("read_set_begin"); + read_set_end_method_id = rb_intern("read_set_end"); + read_struct_begin_method_id = rb_intern("read_struct_begin"); + read_struct_end_method_id = rb_intern("read_struct_end"); + read_field_begin_method_id = rb_intern("read_field_begin"); + read_field_end_method_id = rb_intern("read_field_end"); + keys_method_id = rb_intern("keys"); + entries_method_id = rb_intern("entries"); + name_method_id = rb_intern("name"); + sort_method_id = rb_intern("sort"); + write_field_stop_method_id = rb_intern("write_field_stop"); + skip_method_id = rb_intern("skip"); + write_method_id = rb_intern("write"); + read_all_method_id = rb_intern("read_all"); + native_qmark_method_id = rb_intern("native?"); + + // constant ids + fields_const_id = rb_intern("FIELDS"); + transport_ivar_id = rb_intern("@trans"); + strict_read_ivar_id = rb_intern("@strict_read"); + strict_write_ivar_id = rb_intern("@strict_write"); + + // cached symbols + type_sym = ID2SYM(rb_intern("type")); + name_sym = ID2SYM(rb_intern("name")); + key_sym = ID2SYM(rb_intern("key")); + value_sym = ID2SYM(rb_intern("value")); + element_sym = ID2SYM(rb_intern("element")); + class_sym = ID2SYM(rb_intern("class")); + + Init_protocol(); + Init_struct(); + Init_binary_protocol_accelerated(); + Init_compact_protocol(); + Init_memory_buffer(); +} diff --git a/lib/rb/lib/thrift.rb b/lib/rb/lib/thrift.rb new file mode 100644 index 00000000..88562e13 --- /dev/null +++ b/lib/rb/lib/thrift.rb @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +$:.unshift File.dirname(__FILE__) + +require 'thrift/core_ext' +require 'thrift/exceptions' +require 'thrift/types' +require 'thrift/processor' +require 'thrift/client' +require 'thrift/struct' + +# serializer +require 'thrift/serializer/serializer' +require 'thrift/serializer/deserializer' + +# protocol +require 'thrift/protocol/base_protocol' +require 'thrift/protocol/binary_protocol' +require 'thrift/protocol/binary_protocol_accelerated' +require 'thrift/protocol/compact_protocol' + +# transport +require 'thrift/transport/base_transport' +require 'thrift/transport/base_server_transport' +require 'thrift/transport/socket' +require 'thrift/transport/server_socket' +require 'thrift/transport/unix_socket' +require 'thrift/transport/unix_server_socket' +require 'thrift/transport/buffered_transport' +require 'thrift/transport/framed_transport' +require 'thrift/transport/http_client_transport' +require 'thrift/transport/io_stream_transport' +require 'thrift/transport/memory_buffer_transport' + +# server +require 'thrift/server/base_server' +require 'thrift/server/nonblocking_server' +require 'thrift/server/simple_server' +require 'thrift/server/threaded_server' +require 'thrift/server/thread_pool_server' + +require 'thrift/thrift_native' \ No newline at end of file diff --git a/lib/rb/lib/thrift/client.rb b/lib/rb/lib/thrift/client.rb new file mode 100644 index 00000000..5b30f015 --- /dev/null +++ b/lib/rb/lib/thrift/client.rb @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + module Client + def initialize(iprot, oprot=nil) + @iprot = iprot + @oprot = oprot || iprot + @seqid = 0 + end + + def send_message(name, args_class, args = {}) + @oprot.write_message_begin(name, MessageTypes::CALL, @seqid) + data = args_class.new + args.each do |k, v| + data.send("#{k.to_s}=", v) + end + begin + data.write(@oprot) + rescue StandardError => e + @oprot.trans.close + raise e + end + @oprot.write_message_end + @oprot.trans.flush + end + + def receive_message(result_klass) + fname, mtype, rseqid = @iprot.read_message_begin + handle_exception(mtype) + result = result_klass.new + result.read(@iprot) + @iprot.read_message_end + result + end + + def handle_exception(mtype) + if mtype == MessageTypes::EXCEPTION + x = ApplicationException.new + x.read(@iprot) + @iprot.read_message_end + raise x + end + end + end +end diff --git a/lib/rb/lib/thrift/core_ext.rb b/lib/rb/lib/thrift/core_ext.rb new file mode 100644 index 00000000..f763cd53 --- /dev/null +++ b/lib/rb/lib/thrift/core_ext.rb @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +Dir[File.dirname(__FILE__) + "/core_ext/*.rb"].each do |file| + name = File.basename(file, '.rb') + require "thrift/core_ext/#{name}" +end diff --git a/lib/rb/lib/thrift/core_ext/fixnum.rb b/lib/rb/lib/thrift/core_ext/fixnum.rb new file mode 100644 index 00000000..b4fc90dd --- /dev/null +++ b/lib/rb/lib/thrift/core_ext/fixnum.rb @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Versions of ruby pre 1.8.7 do not have an .ord method available in the Fixnum +# class. +# +if RUBY_VERSION < "1.8.7" + class Fixnum + def ord + self + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/exceptions.rb b/lib/rb/lib/thrift/exceptions.rb new file mode 100644 index 00000000..dda70894 --- /dev/null +++ b/lib/rb/lib/thrift/exceptions.rb @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + class Exception < StandardError + def initialize(message) + super + @message = message + end + + attr_reader :message + end + + class ApplicationException < Exception + + UNKNOWN = 0 + UNKNOWN_METHOD = 1 + INVALID_MESSAGE_TYPE = 2 + WRONG_METHOD_NAME = 3 + BAD_SEQUENCE_ID = 4 + MISSING_RESULT = 5 + + attr_reader :type + + def initialize(type=UNKNOWN, message=nil) + super(message) + @type = type + end + + def read(iprot) + iprot.read_struct_begin + while true + fname, ftype, fid = iprot.read_field_begin + if ftype == Types::STOP + break + end + if fid == 1 and ftype == Types::STRING + @message = iprot.read_string + elsif fid == 2 and ftype == Types::I32 + @type = iprot.read_i32 + else + iprot.skip(ftype) + end + iprot.read_field_end + end + iprot.read_struct_end + end + + def write(oprot) + oprot.write_struct_begin('Thrift::ApplicationException') + unless @message.nil? + oprot.write_field_begin('message', Types::STRING, 1) + oprot.write_string(@message) + oprot.write_field_end + end + unless @type.nil? + oprot.write_field_begin('type', Types::I32, 2) + oprot.write_i32(@type) + oprot.write_field_end + end + oprot.write_field_stop + oprot.write_struct_end + end + + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/processor.rb b/lib/rb/lib/thrift/processor.rb new file mode 100644 index 00000000..5d9e0a11 --- /dev/null +++ b/lib/rb/lib/thrift/processor.rb @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + module Processor + def initialize(handler) + @handler = handler + end + + def process(iprot, oprot) + name, type, seqid = iprot.read_message_begin + if respond_to?("process_#{name}") + send("process_#{name}", seqid, iprot, oprot) + true + else + iprot.skip(Types::STRUCT) + iprot.read_message_end + x = ApplicationException.new(ApplicationException::UNKNOWN_METHOD, 'Unknown function '+name) + oprot.write_message_begin(name, MessageTypes::EXCEPTION, seqid) + x.write(oprot) + oprot.write_message_end + oprot.trans.flush + false + end + end + + def read_args(iprot, args_class) + args = args_class.new + args.read(iprot) + iprot.read_message_end + args + end + + def write_result(result, oprot, name, seqid) + oprot.write_message_begin(name, MessageTypes::REPLY, seqid) + result.write(oprot) + oprot.write_message_end + oprot.trans.flush + end + end +end diff --git a/lib/rb/lib/thrift/protocol/base_protocol.rb b/lib/rb/lib/thrift/protocol/base_protocol.rb new file mode 100644 index 00000000..b19909d5 --- /dev/null +++ b/lib/rb/lib/thrift/protocol/base_protocol.rb @@ -0,0 +1,290 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# this require is to make generated struct definitions happy +require 'set' + +module Thrift + class ProtocolException < Exception + + UNKNOWN = 0 + INVALID_DATA = 1 + NEGATIVE_SIZE = 2 + SIZE_LIMIT = 3 + BAD_VERSION = 4 + + attr_reader :type + + def initialize(type=UNKNOWN, message=nil) + super(message) + @type = type + end + end + + class BaseProtocol + + attr_reader :trans + + def initialize(trans) + @trans = trans + end + + def native? + puts "wrong method is being called!" + false + end + + def write_message_begin(name, type, seqid) + raise NotImplementedError + end + + def write_message_end; nil; end + + def write_struct_begin(name) + raise NotImplementedError + end + + def write_struct_end; nil; end + + def write_field_begin(name, type, id) + raise NotImplementedError + end + + def write_field_end; nil; end + + def write_field_stop + raise NotImplementedError + end + + def write_map_begin(ktype, vtype, size) + raise NotImplementedError + end + + def write_map_end; nil; end + + def write_list_begin(etype, size) + raise NotImplementedError + end + + def write_list_end; nil; end + + def write_set_begin(etype, size) + raise NotImplementedError + end + + def write_set_end; nil; end + + def write_bool(bool) + raise NotImplementedError + end + + def write_byte(byte) + raise NotImplementedError + end + + def write_i16(i16) + raise NotImplementedError + end + + def write_i32(i32) + raise NotImplementedError + end + + def write_i64(i64) + raise NotImplementedError + end + + def write_double(dub) + raise NotImplementedError + end + + def write_string(str) + raise NotImplementedError + end + + def read_message_begin + raise NotImplementedError + end + + def read_message_end; nil; end + + def read_struct_begin + raise NotImplementedError + end + + def read_struct_end; nil; end + + def read_field_begin + raise NotImplementedError + end + + def read_field_end; nil; end + + def read_map_begin + raise NotImplementedError + end + + def read_map_end; nil; end + + def read_list_begin + raise NotImplementedError + end + + def read_list_end; nil; end + + def read_set_begin + raise NotImplementedError + end + + def read_set_end; nil; end + + def read_bool + raise NotImplementedError + end + + def read_byte + raise NotImplementedError + end + + def read_i16 + raise NotImplementedError + end + + def read_i32 + raise NotImplementedError + end + + def read_i64 + raise NotImplementedError + end + + def read_double + raise NotImplementedError + end + + def read_string + raise NotImplementedError + end + + def write_field(name, type, fid, value) + write_field_begin(name, type, fid) + write_type(type, value) + write_field_end + end + + def write_type(type, value) + case type + when Types::BOOL + write_bool(value) + when Types::BYTE + write_byte(value) + when Types::DOUBLE + write_double(value) + when Types::I16 + write_i16(value) + when Types::I32 + write_i32(value) + when Types::I64 + write_i64(value) + when Types::STRING + write_string(value) + when Types::STRUCT + value.write(self) + else + raise NotImplementedError + end + end + + def read_type(type) + case type + when Types::BOOL + read_bool + when Types::BYTE + read_byte + when Types::DOUBLE + read_double + when Types::I16 + read_i16 + when Types::I32 + read_i32 + when Types::I64 + read_i64 + when Types::STRING + read_string + else + raise NotImplementedError + end + end + + def skip(type) + case type + when Types::STOP + nil + when Types::BOOL + read_bool + when Types::BYTE + read_byte + when Types::I16 + read_i16 + when Types::I32 + read_i32 + when Types::I64 + read_i64 + when Types::DOUBLE + read_double + when Types::STRING + read_string + when Types::STRUCT + read_struct_begin + while true + name, type, id = read_field_begin + break if type == Types::STOP + skip(type) + read_field_end + end + read_struct_end + when Types::MAP + ktype, vtype, size = read_map_begin + size.times do + skip(ktype) + skip(vtype) + end + read_map_end + when Types::SET + etype, size = read_set_begin + size.times do + skip(etype) + end + read_set_end + when Types::LIST + etype, size = read_list_begin + size.times do + skip(etype) + end + read_list_end + end + end + end + + class BaseProtocolFactory + def get_protocol(trans) + raise NotImplementedError + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/protocol/binary_protocol.rb b/lib/rb/lib/thrift/protocol/binary_protocol.rb new file mode 100644 index 00000000..04d149ac --- /dev/null +++ b/lib/rb/lib/thrift/protocol/binary_protocol.rb @@ -0,0 +1,225 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + class BinaryProtocol < BaseProtocol + VERSION_MASK = 0xffff0000 + VERSION_1 = 0x80010000 + TYPE_MASK = 0x000000ff + + attr_reader :strict_read, :strict_write + + def initialize(trans, strict_read=true, strict_write=true) + super(trans) + @strict_read = strict_read + @strict_write = strict_write + end + + def write_message_begin(name, type, seqid) + # this is necessary because we added (needed) bounds checking to + # write_i32, and 0x80010000 is too big for that. + if strict_write + write_i16(VERSION_1 >> 16) + write_i16(type) + write_string(name) + write_i32(seqid) + else + write_string(name) + write_byte(type) + write_i32(seqid) + end + end + + def write_struct_begin(name); nil; end + + def write_field_begin(name, type, id) + write_byte(type) + write_i16(id) + end + + def write_field_stop + write_byte(Thrift::Types::STOP) + end + + def write_map_begin(ktype, vtype, size) + write_byte(ktype) + write_byte(vtype) + write_i32(size) + end + + def write_list_begin(etype, size) + write_byte(etype) + write_i32(size) + end + + def write_set_begin(etype, size) + write_byte(etype) + write_i32(size) + end + + def write_bool(bool) + write_byte(bool ? 1 : 0) + end + + def write_byte(byte) + raise RangeError if byte < -2**31 || byte >= 2**32 + trans.write([byte].pack('c')) + end + + def write_i16(i16) + trans.write([i16].pack('n')) + end + + def write_i32(i32) + raise RangeError if i32 < -2**31 || i32 >= 2**31 + trans.write([i32].pack('N')) + end + + def write_i64(i64) + raise RangeError if i64 < -2**63 || i64 >= 2**64 + hi = i64 >> 32 + lo = i64 & 0xffffffff + trans.write([hi, lo].pack('N2')) + end + + def write_double(dub) + trans.write([dub].pack('G')) + end + + def write_string(str) + write_i32(str.length) + trans.write(str) + end + + def read_message_begin + version = read_i32 + if version < 0 + if (version & VERSION_MASK != VERSION_1) + raise ProtocolException.new(ProtocolException::BAD_VERSION, 'Missing version identifier') + end + type = version & TYPE_MASK + name = read_string + seqid = read_i32 + [name, type, seqid] + else + if strict_read + raise ProtocolException.new(ProtocolException::BAD_VERSION, 'No version identifier, old protocol client?') + end + name = trans.read_all(version) + type = read_byte + seqid = read_i32 + [name, type, seqid] + end + end + + def read_struct_begin; nil; end + + def read_field_begin + type = read_byte + if (type == Types::STOP) + [nil, type, 0] + else + id = read_i16 + [nil, type, id] + end + end + + def read_map_begin + ktype = read_byte + vtype = read_byte + size = read_i32 + [ktype, vtype, size] + end + + def read_list_begin + etype = read_byte + size = read_i32 + [etype, size] + end + + def read_set_begin + etype = read_byte + size = read_i32 + [etype, size] + end + + def read_bool + byte = read_byte + byte != 0 + end + + def read_byte + dat = trans.read_all(1) + val = dat[0].ord + if (val > 0x7f) + val = 0 - ((val - 1) ^ 0xff) + end + val + end + + def read_i16 + dat = trans.read_all(2) + val, = dat.unpack('n') + if (val > 0x7fff) + val = 0 - ((val - 1) ^ 0xffff) + end + val + end + + def read_i32 + dat = trans.read_all(4) + val, = dat.unpack('N') + if (val > 0x7fffffff) + val = 0 - ((val - 1) ^ 0xffffffff) + end + val + end + + def read_i64 + dat = trans.read_all(8) + hi, lo = dat.unpack('N2') + if (hi > 0x7fffffff) + hi ^= 0xffffffff + lo ^= 0xffffffff + 0 - (hi << 32) - lo - 1 + else + (hi << 32) + lo + end + end + + def read_double + dat = trans.read_all(8) + val = dat.unpack('G').first + val + end + + def read_string + sz = read_i32 + dat = trans.read_all(sz) + dat + end + + end + + class BinaryProtocolFactory < BaseProtocolFactory + def get_protocol(trans) + return Thrift::BinaryProtocol.new(trans) + end + end +end diff --git a/lib/rb/lib/thrift/protocol/binary_protocol_accelerated.rb b/lib/rb/lib/thrift/protocol/binary_protocol_accelerated.rb new file mode 100644 index 00000000..eaf64f6b --- /dev/null +++ b/lib/rb/lib/thrift/protocol/binary_protocol_accelerated.rb @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +=begin +The only change required for a transport to support BinaryProtocolAccelerated is to implement 2 methods: + * borrow(size), which takes an optional argument and returns atleast _size_ bytes from the transport, + or the default buffer size if no argument is given + * consume!(size), which removes size bytes from the front of the buffer + +See MemoryBuffer and BufferedTransport for examples. +=end + +module Thrift + class BinaryProtocolAcceleratedFactory < BaseProtocolFactory + def get_protocol(trans) + BinaryProtocolAccelerated.new(trans) + end + end +end diff --git a/lib/rb/lib/thrift/protocol/compact_protocol.rb b/lib/rb/lib/thrift/protocol/compact_protocol.rb new file mode 100644 index 00000000..c8f43655 --- /dev/null +++ b/lib/rb/lib/thrift/protocol/compact_protocol.rb @@ -0,0 +1,422 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + class CompactProtocol < BaseProtocol + + PROTOCOL_ID = [0x82].pack('c').unpack('c').first + VERSION = 1 + VERSION_MASK = 0x1f + TYPE_MASK = 0xE0 + TYPE_SHIFT_AMOUNT = 5 + + TSTOP = ["", Types::STOP, 0] + + # + # All of the on-wire type codes. + # + class CompactTypes + BOOLEAN_TRUE = 0x01 + BOOLEAN_FALSE = 0x02 + BYTE = 0x03 + I16 = 0x04 + I32 = 0x05 + I64 = 0x06 + DOUBLE = 0x07 + BINARY = 0x08 + LIST = 0x09 + SET = 0x0A + MAP = 0x0B + STRUCT = 0x0C + + def self.is_bool_type?(b) + (b & 0x0f) == BOOLEAN_TRUE || (b & 0x0f) == BOOLEAN_FALSE + end + + COMPACT_TO_TTYPE = { + Types::STOP => Types::STOP, + BOOLEAN_FALSE => Types::BOOL, + BOOLEAN_TRUE => Types::BOOL, + BYTE => Types::BYTE, + I16 => Types::I16, + I32 => Types::I32, + I64 => Types::I64, + DOUBLE => Types::DOUBLE, + BINARY => Types::STRING, + LIST => Types::LIST, + SET => Types::SET, + MAP => Types::MAP, + STRUCT => Types::STRUCT + } + + TTYPE_TO_COMPACT = { + Types::STOP => Types::STOP, + Types::BOOL => BOOLEAN_TRUE, + Types::BYTE => BYTE, + Types::I16 => I16, + Types::I32 => I32, + Types::I64 => I64, + Types::DOUBLE => DOUBLE, + Types::STRING => BINARY, + Types::LIST => LIST, + Types::SET => SET, + Types::MAP => MAP, + Types::STRUCT => STRUCT + } + + def self.get_ttype(compact_type) + val = COMPACT_TO_TTYPE[compact_type & 0x0f] + raise "don't know what type: #{compact_type & 0x0f}" unless val + val + end + + def self.get_compact_type(ttype) + val = TTYPE_TO_COMPACT[ttype] + raise "don't know what type: #{ttype & 0x0f}" unless val + val + end + end + + def initialize(transport) + super(transport) + + @last_field = [0] + @boolean_value = nil + end + + def write_message_begin(name, type, seqid) + write_byte(PROTOCOL_ID) + write_byte((VERSION & VERSION_MASK) | ((type << TYPE_SHIFT_AMOUNT) & TYPE_MASK)) + write_varint32(seqid) + write_string(name) + nil + end + + def write_struct_begin(name) + @last_field.push(0) + nil + end + + def write_struct_end + @last_field.pop + nil + end + + def write_field_begin(name, type, id) + if type == Types::BOOL + # we want to possibly include the value, so we'll wait. + @boolean_field = [type, id] + else + write_field_begin_internal(type, id) + end + nil + end + + # + # The workhorse of writeFieldBegin. It has the option of doing a + # 'type override' of the type header. This is used specifically in the + # boolean field case. + # + def write_field_begin_internal(type, id, type_override=nil) + last_id = @last_field.pop + + # if there's a type override, use that. + typeToWrite = type_override || CompactTypes.get_compact_type(type) + + # check if we can use delta encoding for the field id + if id > last_id && id - last_id <= 15 + # write them together + write_byte((id - last_id) << 4 | typeToWrite) + else + # write them separate + write_byte(typeToWrite) + write_i16(id) + end + + @last_field.push(id) + nil + end + + def write_field_stop + write_byte(Types::STOP) + end + + def write_map_begin(ktype, vtype, size) + if (size == 0) + write_byte(0) + else + write_varint32(size) + write_byte(CompactTypes.get_compact_type(ktype) << 4 | CompactTypes.get_compact_type(vtype)) + end + end + + def write_list_begin(etype, size) + write_collection_begin(etype, size) + end + + def write_set_begin(etype, size) + write_collection_begin(etype, size); + end + + def write_bool(bool) + type = bool ? CompactTypes::BOOLEAN_TRUE : CompactTypes::BOOLEAN_FALSE + unless @boolean_field.nil? + # we haven't written the field header yet + write_field_begin_internal(@boolean_field.first, @boolean_field.last, type) + @boolean_field = nil + else + # we're not part of a field, so just write the value. + write_byte(type) + end + end + + def write_byte(byte) + @trans.write([byte].pack('c')) + end + + def write_i16(i16) + write_varint32(int_to_zig_zag(i16)) + end + + def write_i32(i32) + write_varint32(int_to_zig_zag(i32)) + end + + def write_i64(i64) + write_varint64(long_to_zig_zag(i64)) + end + + def write_double(dub) + @trans.write([dub].pack("G").reverse) + end + + def write_string(str) + write_varint32(str.length) + @trans.write(str) + end + + def read_message_begin + protocol_id = read_byte() + if protocol_id != PROTOCOL_ID + raise ProtocolException.new("Expected protocol id #{PROTOCOL_ID} but got #{protocol_id}") + end + + version_and_type = read_byte() + version = version_and_type & VERSION_MASK + if (version != VERSION) + raise ProtocolException.new("Expected version #{VERSION} but got #{version}"); + end + + type = (version_and_type >> TYPE_SHIFT_AMOUNT) & 0x03 + seqid = read_varint32() + messageName = read_string() + [messageName, type, seqid] + end + + def read_struct_begin + @last_field.push(0) + "" + end + + def read_struct_end + @last_field.pop() + nil + end + + def read_field_begin + type = read_byte() + + # if it's a stop, then we can return immediately, as the struct is over. + if (type & 0x0f) == Types::STOP + TSTOP + else + field_id = nil + + # mask off the 4 MSB of the type header. it could contain a field id delta. + modifier = (type & 0xf0) >> 4 + if modifier == 0 + # not a delta. look ahead for the zigzag varint field id. + field_id = read_i16() + else + # has a delta. add the delta to the last read field id. + field_id = @last_field.pop + modifier + end + + # if this happens to be a boolean field, the value is encoded in the type + if CompactTypes.is_bool_type?(type) + # save the boolean value in a special instance variable. + @bool_value = (type & 0x0f) == CompactTypes::BOOLEAN_TRUE + end + + # push the new field onto the field stack so we can keep the deltas going. + @last_field.push(field_id) + ["", CompactTypes.get_ttype(type & 0x0f), field_id] + end + end + + def read_map_begin + size = read_varint32() + key_and_value_type = size == 0 ? 0 : read_byte() + [CompactTypes.get_ttype(key_and_value_type >> 4), CompactTypes.get_ttype(key_and_value_type & 0xf), size] + end + + def read_list_begin + size_and_type = read_byte() + size = (size_and_type >> 4) & 0x0f + if size == 15 + size = read_varint32() + end + type = CompactTypes.get_ttype(size_and_type) + [type, size] + end + + def read_set_begin + read_list_begin + end + + def read_bool + unless @bool_value.nil? + bv = @bool_value + @bool_value = nil + bv + else + read_byte() == CompactTypes::BOOLEAN_TRUE + end + end + + def read_byte + dat = trans.read_all(1) + val = dat[0] + if (val > 0x7f) + val = 0 - ((val - 1) ^ 0xff) + end + val + end + + def read_i16 + zig_zag_to_int(read_varint32()) + end + + def read_i32 + zig_zag_to_int(read_varint32()) + end + + def read_i64 + zig_zag_to_long(read_varint64()) + end + + def read_double + dat = trans.read_all(8) + val = dat.reverse.unpack('G').first + val + end + + def read_string + size = read_varint32() + trans.read_all(size) + end + + + private + + # + # Abstract method for writing the start of lists and sets. List and sets on + # the wire differ only by the type indicator. + # + def write_collection_begin(elem_type, size) + if size <= 14 + write_byte(size << 4 | CompactTypes.get_compact_type(elem_type)) + else + write_byte(0xf0 | CompactTypes.get_compact_type(elem_type)) + write_varint32(size) + end + end + + def write_varint32(n) + # int idx = 0; + while true + if (n & ~0x7F) == 0 + # i32buf[idx++] = (byte)n; + write_byte(n) + break + # return; + else + # i32buf[idx++] = (byte)((n & 0x7F) | 0x80); + write_byte((n & 0x7F) | 0x80) + n = n >> 7 + end + end + # trans_.write(i32buf, 0, idx); + end + + SEVEN_BIT_MASK = 0x7F + EVERYTHING_ELSE_MASK = ~SEVEN_BIT_MASK + + def write_varint64(n) + while true + if (n & EVERYTHING_ELSE_MASK) == 0 #TODO need to find a way to make this into a long... + write_byte(n) + break + else + write_byte((n & SEVEN_BIT_MASK) | 0x80) + n >>= 7 + end + end + end + + def read_varint32() + read_varint64() + end + + def read_varint64() + shift = 0 + result = 0 + while true + b = read_byte() + result |= (b & 0x7f) << shift + break if (b & 0x80) != 0x80 + shift += 7 + end + result + end + + def int_to_zig_zag(n) + (n << 1) ^ (n >> 31) + end + + def long_to_zig_zag(l) + # puts "zz encoded #{l} to #{(l << 1) ^ (l >> 63)}" + (l << 1) ^ (l >> 63) + end + + def zig_zag_to_int(n) + (n >> 1) ^ -(n & 1) + end + + def zig_zag_to_long(n) + (n >> 1) ^ -(n & 1) + end + end + + class CompactProtocolFactory < BaseProtocolFactory + def get_protocol(trans) + CompactProtocol.new(trans) + end + end +end diff --git a/lib/rb/lib/thrift/serializer/deserializer.rb b/lib/rb/lib/thrift/serializer/deserializer.rb new file mode 100644 index 00000000..d2ee325a --- /dev/null +++ b/lib/rb/lib/thrift/serializer/deserializer.rb @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + class Deserializer + def initialize(protocol_factory = BinaryProtocolFactory.new) + @transport = MemoryBufferTransport.new + @protocol = protocol_factory.get_protocol(@transport) + end + + def deserialize(base, buffer) + @transport.reset_buffer(buffer) + base.read(@protocol) + base + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/serializer/serializer.rb b/lib/rb/lib/thrift/serializer/serializer.rb new file mode 100644 index 00000000..22316395 --- /dev/null +++ b/lib/rb/lib/thrift/serializer/serializer.rb @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + class Serializer + def initialize(protocol_factory = BinaryProtocolFactory.new) + @transport = MemoryBufferTransport.new + @protocol = protocol_factory.get_protocol(@transport) + end + + def serialize(base) + @transport.reset_buffer + base.write(@protocol) + @transport.read(@transport.available) + end + end +end + diff --git a/lib/rb/lib/thrift/server/base_server.rb b/lib/rb/lib/thrift/server/base_server.rb new file mode 100644 index 00000000..1ee12133 --- /dev/null +++ b/lib/rb/lib/thrift/server/base_server.rb @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + class BaseServer + def initialize(processor, server_transport, transport_factory=nil, protocol_factory=nil) + @processor = processor + @server_transport = server_transport + @transport_factory = transport_factory ? transport_factory : Thrift::BaseTransportFactory.new + @protocol_factory = protocol_factory ? protocol_factory : Thrift::BinaryProtocolFactory.new + end + + def serve; nil; end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/server/mongrel_http_server.rb b/lib/rb/lib/thrift/server/mongrel_http_server.rb new file mode 100644 index 00000000..84eacf0d --- /dev/null +++ b/lib/rb/lib/thrift/server/mongrel_http_server.rb @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'mongrel' + +## Sticks a service on a URL, using mongrel to do the HTTP work +module Thrift + class MongrelHTTPServer < BaseServer + class Handler < Mongrel::HttpHandler + def initialize(processor, protocol_factory) + @processor = processor + @protocol_factory = protocol_factory + end + + def process(request, response) + if request.params["REQUEST_METHOD"] == "POST" + response.start(200) do |head, out| + head["Content-Type"] = "application/x-thrift" + transport = IOStreamTransport.new request.body, out + protocol = @protocol_factory.get_protocol transport + @processor.process protocol, protocol + end + else + response.start(404) { } + end + end + end + + def initialize(processor, opts={}) + port = opts[:port] || 80 + ip = opts[:ip] || "0.0.0.0" + path = opts[:path] || "" + protocol_factory = opts[:protocol_factory] || BinaryProtocolFactory.new + @server = Mongrel::HttpServer.new ip, port + @server.register "/#{path}", Handler.new(processor, protocol_factory) + end + + def serve + @server.run.join + end + end +end diff --git a/lib/rb/lib/thrift/server/nonblocking_server.rb b/lib/rb/lib/thrift/server/nonblocking_server.rb new file mode 100644 index 00000000..5425f6de --- /dev/null +++ b/lib/rb/lib/thrift/server/nonblocking_server.rb @@ -0,0 +1,296 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'logger' +require 'thread' + +module Thrift + # this class expects to always use a FramedTransport for reading messages + class NonblockingServer < BaseServer + def initialize(processor, server_transport, transport_factory=nil, protocol_factory=nil, num=20, logger=nil) + super(processor, server_transport, transport_factory, protocol_factory) + @num_threads = num + if logger.nil? + @logger = Logger.new(STDERR) + @logger.level = Logger::WARN + else + @logger = logger + end + @shutdown_semaphore = Mutex.new + @transport_semaphore = Mutex.new + end + + def serve + @logger.info "Starting #{self}" + @server_transport.listen + @io_manager = start_io_manager + + begin + loop do + break if @server_transport.closed? + rd, = select([@server_transport], nil, nil, 0.1) + next if rd.nil? + socket = @server_transport.accept + @logger.debug "Accepted socket: #{socket.inspect}" + @io_manager.add_connection socket + end + rescue IOError => e + end + # we must be shutting down + @logger.info "#{self} is shutting down, goodbye" + ensure + @transport_semaphore.synchronize do + @server_transport.close + end + @io_manager.ensure_closed unless @io_manager.nil? + end + + def shutdown(timeout = 0, block = true) + @shutdown_semaphore.synchronize do + return if @is_shutdown + @is_shutdown = true + end + # nonblocking is intended for calling from within a Handler + # but we can't change the order of operations here, so lets thread + shutdown_proc = lambda do + @io_manager.shutdown(timeout) + @transport_semaphore.synchronize do + @server_transport.close # this will break the accept loop + end + end + if block + shutdown_proc.call + else + Thread.new &shutdown_proc + end + end + + private + + def start_io_manager + iom = IOManager.new(@processor, @server_transport, @transport_factory, @protocol_factory, @num_threads, @logger) + iom.spawn + iom + end + + class IOManager # :nodoc: + DEFAULT_BUFFER = 2**20 + + def initialize(processor, server_transport, transport_factory, protocol_factory, num, logger) + @processor = processor + @server_transport = server_transport + @transport_factory = transport_factory + @protocol_factory = protocol_factory + @num_threads = num + @logger = logger + @connections = [] + @buffers = Hash.new { |h,k| h[k] = '' } + @signal_queue = Queue.new + @signal_pipes = IO.pipe + @signal_pipes[1].sync = true + @worker_queue = Queue.new + @shutdown_queue = Queue.new + end + + def add_connection(socket) + signal [:connection, socket] + end + + def spawn + @iom_thread = Thread.new do + @logger.debug "Starting #{self}" + run + end + end + + def shutdown(timeout = 0) + @logger.debug "#{self} is shutting down workers" + @worker_queue.clear + @num_threads.times { @worker_queue.push [:shutdown] } + signal [:shutdown, timeout] + @shutdown_queue.pop + @signal_pipes[0].close + @signal_pipes[1].close + @logger.debug "#{self} is shutting down, goodbye" + end + + def ensure_closed + kill_worker_threads if @worker_threads + @iom_thread.kill + end + + private + + def run + spin_worker_threads + + loop do + rd, = select([@signal_pipes[0], *@connections]) + if rd.delete @signal_pipes[0] + break if read_signals == :shutdown + end + rd.each do |fd| + if fd.handle.eof? + remove_connection fd + else + read_connection fd + end + end + end + join_worker_threads(@shutdown_timeout) + ensure + @shutdown_queue.push :shutdown + end + + def read_connection(fd) + @buffers[fd] << fd.read(DEFAULT_BUFFER) + frame = slice_frame!(@buffers[fd]) + if frame + @logger.debug "#{self} is processing a frame" + @worker_queue.push [:frame, fd, frame] + end + end + + def spin_worker_threads + @logger.debug "#{self} is spinning up worker threads" + @worker_threads = [] + @num_threads.times do + @worker_threads << spin_thread + end + end + + def spin_thread + Worker.new(@processor, @transport_factory, @protocol_factory, @logger, @worker_queue).spawn + end + + def signal(msg) + @signal_queue << msg + @signal_pipes[1].write " " + end + + def read_signals + # clear the signal pipe + # note that since read_nonblock is broken in jruby, + # we can only read up to a set number of signals at once + sigstr = @signal_pipes[0].readpartial(1024) + # now read the signals + begin + sigstr.length.times do + signal, obj = @signal_queue.pop(true) + case signal + when :connection + @connections << obj + when :shutdown + @shutdown_timeout = obj + return :shutdown + end + end + rescue ThreadError + # out of signals + # note that in a perfect world this would never happen, since we're + # only reading the number of signals pushed on the pipe, but given the lack + # of locks, in theory we could clear the pipe/queue while a new signal is being + # placed on the pipe, at which point our next read_signals would hit this error + end + end + + def remove_connection(fd) + # don't explicitly close it, a thread may still be writing to it + @connections.delete fd + @buffers.delete fd + end + + def join_worker_threads(shutdown_timeout) + start = Time.now + @worker_threads.each do |t| + if shutdown_timeout > 0 + timeout = (start + shutdown_timeout) - Time.now + break if timeout <= 0 + t.join(timeout) + else + t.join + end + end + kill_worker_threads + end + + def kill_worker_threads + @worker_threads.each do |t| + t.kill if t.status + end + @worker_threads.clear + end + + def slice_frame!(buf) + if buf.length >= 4 + size = buf.unpack('N').first + if buf.length >= size + 4 + buf.slice!(0, size + 4) + else + nil + end + else + nil + end + end + + class Worker # :nodoc: + def initialize(processor, transport_factory, protocol_factory, logger, queue) + @processor = processor + @transport_factory = transport_factory + @protocol_factory = protocol_factory + @logger = logger + @queue = queue + end + + def spawn + Thread.new do + @logger.debug "#{self} is spawning" + run + end + end + + private + + def run + loop do + cmd, *args = @queue.pop + case cmd + when :shutdown + @logger.debug "#{self} is shutting down, goodbye" + break + when :frame + fd, frame = args + begin + otrans = @transport_factory.get_transport(fd) + oprot = @protocol_factory.get_protocol(otrans) + membuf = MemoryBufferTransport.new(frame) + itrans = @transport_factory.get_transport(membuf) + iprot = @protocol_factory.get_protocol(itrans) + @processor.process(iprot, oprot) + rescue => e + @logger.error "#{Thread.current.inspect} raised error: #{e.inspect}\n#{e.backtrace.join("\n")}" + end + end + end + end + end + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/server/simple_server.rb b/lib/rb/lib/thrift/server/simple_server.rb new file mode 100644 index 00000000..21e86592 --- /dev/null +++ b/lib/rb/lib/thrift/server/simple_server.rb @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + class SimpleServer < BaseServer + def serve + begin + @server_transport.listen + loop do + client = @server_transport.accept + trans = @transport_factory.get_transport(client) + prot = @protocol_factory.get_protocol(trans) + begin + loop do + @processor.process(prot, prot) + end + rescue Thrift::TransportException, Thrift::ProtocolException + ensure + trans.close + end + end + ensure + @server_transport.close + end + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/server/thread_pool_server.rb b/lib/rb/lib/thrift/server/thread_pool_server.rb new file mode 100644 index 00000000..8cec805a --- /dev/null +++ b/lib/rb/lib/thrift/server/thread_pool_server.rb @@ -0,0 +1,75 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'thread' + +module Thrift + class ThreadPoolServer < BaseServer + def initialize(processor, server_transport, transport_factory=nil, protocol_factory=nil, num=20) + super(processor, server_transport, transport_factory, protocol_factory) + @thread_q = SizedQueue.new(num) + @exception_q = Queue.new + @running = false + end + + ## exceptions that happen in worker threads will be relayed here and + ## must be caught. 'retry' can be used to continue. (threads will + ## continue to run while the exception is being handled.) + def rescuable_serve + Thread.new { serve } unless @running + @running = true + raise @exception_q.pop + end + + ## exceptions that happen in worker threads simply cause that thread + ## to die and another to be spawned in its place. + def serve + @server_transport.listen + + begin + loop do + @thread_q.push(:token) + Thread.new do + begin + loop do + client = @server_transport.accept + trans = @transport_factory.get_transport(client) + prot = @protocol_factory.get_protocol(trans) + begin + loop do + @processor.process(prot, prot) + end + rescue Thrift::TransportException, Thrift::ProtocolException => e + ensure + trans.close + end + end + rescue => e + @exception_q.push(e) + ensure + @thread_q.pop # thread died! + end + end + end + ensure + @server_transport.close + end + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/server/threaded_server.rb b/lib/rb/lib/thrift/server/threaded_server.rb new file mode 100644 index 00000000..a2c917cb --- /dev/null +++ b/lib/rb/lib/thrift/server/threaded_server.rb @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'thread' + +module Thrift + class ThreadedServer < BaseServer + def serve + begin + @server_transport.listen + loop do + client = @server_transport.accept + trans = @transport_factory.get_transport(client) + prot = @protocol_factory.get_protocol(trans) + Thread.new(prot, trans) do |p, t| + begin + loop do + @processor.process(p, p) + end + rescue Thrift::TransportException, Thrift::ProtocolException + ensure + t.close + end + end + end + ensure + @server_transport.close + end + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/struct.rb b/lib/rb/lib/thrift/struct.rb new file mode 100644 index 00000000..01aae56b --- /dev/null +++ b/lib/rb/lib/thrift/struct.rb @@ -0,0 +1,294 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'set' + +module Thrift + module Struct + def initialize(d={}) + # get a copy of the default values to work on, removing defaults in favor of arguments + fields_with_defaults = fields_with_default_values.dup + + # check if the defaults is empty, or if there are no parameters for this + # instantiation, and if so, don't bother overriding defaults. + unless fields_with_defaults.empty? || d.empty? + d.each_key do |name| + fields_with_defaults.delete(name.to_s) + end + end + + # assign all the user-specified arguments + unless d.empty? + d.each do |name, value| + unless name_to_id(name.to_s) + raise Exception, "Unknown key given to #{self.class}.new: #{name}" + end + Thrift.check_type(value, struct_fields[name_to_id(name.to_s)], name) if Thrift.type_checking + instance_variable_set("@#{name}", value) + end + end + + # assign all the default values + unless fields_with_defaults.empty? + fields_with_defaults.each do |name, default_value| + instance_variable_set("@#{name}", (default_value.dup rescue default_value)) + end + end + end + + def fields_with_default_values + fields_with_default_values = self.class.instance_variable_get("@fields_with_default_values") + unless fields_with_default_values + fields_with_default_values = {} + struct_fields.each do |fid, field_def| + unless field_def[:default].nil? + fields_with_default_values[field_def[:name]] = field_def[:default] + end + end + self.class.instance_variable_set("@fields_with_default_values", fields_with_default_values) + end + fields_with_default_values + end + + def name_to_id(name) + names_to_ids = self.class.instance_variable_get("@names_to_ids") + unless names_to_ids + names_to_ids = {} + struct_fields.each do |fid, field_def| + names_to_ids[field_def[:name]] = fid + end + self.class.instance_variable_set("@names_to_ids", names_to_ids) + end + names_to_ids[name] + end + + def each_field + struct_fields.keys.sort.each do |fid| + data = struct_fields[fid] + yield fid, data + end + end + + def inspect(skip_optional_nulls = true) + fields = [] + each_field do |fid, field_info| + name = field_info[:name] + value = instance_variable_get("@#{name}") + unless skip_optional_nulls && field_info[:optional] && value.nil? + fields << "#{name}:#{value.inspect}" + end + end + "<#{self.class} #{fields.join(", ")}>" + end + + def read(iprot) + iprot.read_struct_begin + loop do + fname, ftype, fid = iprot.read_field_begin + break if (ftype == Types::STOP) + handle_message(iprot, fid, ftype) + iprot.read_field_end + end + iprot.read_struct_end + validate + end + + def write(oprot) + validate + oprot.write_struct_begin(self.class.name) + each_field do |fid, field_info| + name = field_info[:name] + type = field_info[:type] + if (value = instance_variable_get("@#{name}")) + if is_container? type + oprot.write_field_begin(name, type, fid) + write_container(oprot, value, field_info) + oprot.write_field_end + else + oprot.write_field(name, type, fid, value) + end + end + end + oprot.write_field_stop + oprot.write_struct_end + end + + def ==(other) + each_field do |fid, field_info| + name = field_info[:name] + return false unless self.instance_variable_get("@#{name}") == other.instance_variable_get("@#{name}") + end + true + end + + def eql?(other) + self.class == other.class && self == other + end + + # for the time being, we're ok with a naive hash. this could definitely be improved upon. + def hash + 0 + end + + def differences(other) + diffs = [] + unless other.is_a?(self.class) + diffs << "Different class!" + else + each_field do |fid, field_info| + name = field_info[:name] + diffs << "#{name} differs!" unless self.instance_variable_get("@#{name}") == other.instance_variable_get("@#{name}") + end + end + diffs + end + + def self.field_accessor(klass, *fields) + fields.each do |field| + klass.send :attr_reader, field + klass.send :define_method, "#{field}=" do |value| + Thrift.check_type(value, klass::FIELDS.values.find { |f| f[:name].to_s == field.to_s }, field) if Thrift.type_checking + instance_variable_set("@#{field}", value) + end + end + end + + protected + + def self.append_features(mod) + if mod.ancestors.include? ::Exception + mod.send :class_variable_set, :'@@__thrift_struct_real_initialize', mod.instance_method(:initialize) + super + # set up our custom initializer so `raise Xception, 'message'` works + mod.send :define_method, :struct_initialize, mod.instance_method(:initialize) + mod.send :define_method, :initialize, mod.instance_method(:exception_initialize) + else + super + end + end + + def exception_initialize(*args, &block) + if args.size == 1 and args.first.is_a? Hash + # looks like it's a regular Struct initialize + method(:struct_initialize).call(args.first) + else + # call the Struct initializer first with no args + # this will set our field default values + method(:struct_initialize).call() + # now give it to the exception + self.class.send(:class_variable_get, :'@@__thrift_struct_real_initialize').bind(self).call(*args, &block) if args.size > 0 + # self.class.instance_method(:initialize).bind(self).call(*args, &block) + end + end + + def handle_message(iprot, fid, ftype) + field = struct_fields[fid] + if field and field[:type] == ftype + value = read_field(iprot, field) + instance_variable_set("@#{field[:name]}", value) + else + iprot.skip(ftype) + end + end + + def read_field(iprot, field = {}) + case field[:type] + when Types::STRUCT + value = field[:class].new + value.read(iprot) + when Types::MAP + key_type, val_type, size = iprot.read_map_begin + value = {} + size.times do + k = read_field(iprot, field_info(field[:key])) + v = read_field(iprot, field_info(field[:value])) + value[k] = v + end + iprot.read_map_end + when Types::LIST + e_type, size = iprot.read_list_begin + value = Array.new(size) do |n| + read_field(iprot, field_info(field[:element])) + end + iprot.read_list_end + when Types::SET + e_type, size = iprot.read_set_begin + value = Set.new + size.times do + element = read_field(iprot, field_info(field[:element])) + value << element + end + iprot.read_set_end + else + value = iprot.read_type(field[:type]) + end + value + end + + def write_data(oprot, value, field) + if is_container? field[:type] + write_container(oprot, value, field) + else + oprot.write_type(field[:type], value) + end + end + + def write_container(oprot, value, field = {}) + case field[:type] + when Types::MAP + oprot.write_map_begin(field[:key][:type], field[:value][:type], value.size) + value.each do |k, v| + write_data(oprot, k, field[:key]) + write_data(oprot, v, field[:value]) + end + oprot.write_map_end + when Types::LIST + oprot.write_list_begin(field[:element][:type], value.size) + value.each do |elem| + write_data(oprot, elem, field[:element]) + end + oprot.write_list_end + when Types::SET + oprot.write_set_begin(field[:element][:type], value.size) + value.each do |v,| # the , is to preserve compatibility with the old Hash-style sets + write_data(oprot, v, field[:element]) + end + oprot.write_set_end + else + raise "Not a container type: #{field[:type]}" + end + end + + CONTAINER_TYPES = [] + CONTAINER_TYPES[Types::LIST] = true + CONTAINER_TYPES[Types::MAP] = true + CONTAINER_TYPES[Types::SET] = true + def is_container?(type) + CONTAINER_TYPES[type] + end + + def field_info(field) + { :type => field[:type], + :class => field[:class], + :key => field[:key], + :value => field[:value], + :element => field[:element] } + end + end +end diff --git a/lib/rb/lib/thrift/thrift_native.rb b/lib/rb/lib/thrift/thrift_native.rb new file mode 100644 index 00000000..4d8df61f --- /dev/null +++ b/lib/rb/lib/thrift/thrift_native.rb @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +begin + require "thrift_native" +rescue LoadError + puts "Unable to load thrift_native extension. Defaulting to pure Ruby libraries." +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/transport/base_server_transport.rb b/lib/rb/lib/thrift/transport/base_server_transport.rb new file mode 100644 index 00000000..68c5af07 --- /dev/null +++ b/lib/rb/lib/thrift/transport/base_server_transport.rb @@ -0,0 +1,37 @@ +# encoding: ascii-8bit +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + class BaseServerTransport + def listen + raise NotImplementedError + end + + def accept + raise NotImplementedError + end + + def close; nil; end + + def closed? + raise NotImplementedError + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/transport/base_transport.rb b/lib/rb/lib/thrift/transport/base_transport.rb new file mode 100644 index 00000000..08a71dab --- /dev/null +++ b/lib/rb/lib/thrift/transport/base_transport.rb @@ -0,0 +1,70 @@ +# encoding: ascii-8bit +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + class TransportException < Exception + UNKNOWN = 0 + NOT_OPEN = 1 + ALREADY_OPEN = 2 + TIMED_OUT = 3 + END_OF_FILE = 4 + + attr_reader :type + + def initialize(type=UNKNOWN, message=nil) + super(message) + @type = type + end + end + + class BaseTransport + def open?; end + + def open; end + + def close; end + + def read(sz) + raise NotImplementedError + end + + def read_all(size) + buf = '' + + while (buf.length < size) + chunk = read(size - buf.length) + buf << chunk + end + + buf + end + + def write(buf); end + alias_method :<<, :write + + def flush; end + end + + class BaseTransportFactory + def get_transport(trans) + return trans + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/transport/buffered_transport.rb b/lib/rb/lib/thrift/transport/buffered_transport.rb new file mode 100644 index 00000000..8dead4e0 --- /dev/null +++ b/lib/rb/lib/thrift/transport/buffered_transport.rb @@ -0,0 +1,77 @@ +# encoding: ascii-8bit +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + class BufferedTransport < BaseTransport + DEFAULT_BUFFER = 4096 + + def initialize(transport) + @transport = transport + @wbuf = '' + @rbuf = '' + @index = 0 + end + + def open? + return @transport.open? + end + + def open + @transport.open + end + + def close + flush + @transport.close + end + + def read(sz) + @index += sz + ret = @rbuf.slice(@index - sz, sz) || '' + + if ret.length == 0 + @rbuf = @transport.read([sz, DEFAULT_BUFFER].max) + @index = sz + ret = @rbuf.slice(0, sz) || '' + end + + ret + end + + def write(buf) + @wbuf << buf + end + + def flush + if @wbuf != '' + @transport.write(@wbuf) + @wbuf = '' + end + + @transport.flush + end + end + + class BufferedTransportFactory < BaseTransportFactory + def get_transport(transport) + return BufferedTransport.new(transport) + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/transport/framed_transport.rb b/lib/rb/lib/thrift/transport/framed_transport.rb new file mode 100644 index 00000000..558af744 --- /dev/null +++ b/lib/rb/lib/thrift/transport/framed_transport.rb @@ -0,0 +1,90 @@ +# encoding: ascii-8bit +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + class FramedTransport < BaseTransport + def initialize(transport, read=true, write=true) + @transport = transport + @rbuf = '' + @wbuf = '' + @read = read + @write = write + @index = 0 + end + + def open? + @transport.open? + end + + def open + @transport.open + end + + def close + @transport.close + end + + def read(sz) + return @transport.read(sz) unless @read + + return '' if sz <= 0 + + read_frame if @index >= @rbuf.length + + @index += sz + @rbuf.slice(@index - sz, sz) || '' + end + + def write(buf,sz=nil) + return @transport.write(buf) unless @write + + @wbuf << (sz ? buf[0...sz] : buf) + end + + # + # Writes the output buffer to the stream in the format of a 4-byte length + # followed by the actual data. + # + def flush + return @transport.flush unless @write + + out = [@wbuf.length].pack('N') + out << @wbuf + @transport.write(out) + @transport.flush + @wbuf = '' + end + + private + + def read_frame + sz = @transport.read_all(4).unpack('N').first + + @index = 0 + @rbuf = @transport.read_all(sz) + end + end + + class FramedTransportFactory < BaseTransportFactory + def get_transport(transport) + return FramedTransport.new(transport) + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/transport/http_client_transport.rb b/lib/rb/lib/thrift/transport/http_client_transport.rb new file mode 100644 index 00000000..a190a983 --- /dev/null +++ b/lib/rb/lib/thrift/transport/http_client_transport.rb @@ -0,0 +1,45 @@ +# encoding: ascii-8bit +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'net/http' +require 'net/https' +require 'uri' +require 'stringio' + +module Thrift + class HTTPClientTransport < BaseTransport + def initialize(url) + @url = URI url + @outbuf = "" + end + + def open?; true end + def read(sz); @inbuf.read sz end + def write(buf); @outbuf << buf end + def flush + http = Net::HTTP.new @url.host, @url.port + http.use_ssl = @url.scheme == "https" + headers = { 'Content-Type' => 'application/x-thrift' } + resp, data = http.post(@url.path, @outbuf, headers) + @inbuf = StringIO.new data + @outbuf = "" + end + end +end diff --git a/lib/rb/lib/thrift/transport/io_stream_transport.rb b/lib/rb/lib/thrift/transport/io_stream_transport.rb new file mode 100644 index 00000000..be348aa0 --- /dev/null +++ b/lib/rb/lib/thrift/transport/io_stream_transport.rb @@ -0,0 +1,39 @@ +# encoding: ascii-8bit +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Very very simple implementation of wrapping two objects, one with a #read +# method and one with a #write method, into a transport for thrift. +# +# Assumes both objects are open, remain open, don't require flushing, etc. +# +module Thrift + class IOStreamTransport < BaseTransport + def initialize(input, output) + @input = input + @output = output + end + + def open?; not @input.closed? or not @output.closed? end + def read(sz); @input.read(sz) end + def write(buf); @output.write(buf) end + def close; @input.close; @output.close end + def to_io; @input end # we're assuming this is used in a IO.select for reading + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/transport/memory_buffer_transport.rb b/lib/rb/lib/thrift/transport/memory_buffer_transport.rb new file mode 100644 index 00000000..33d732d1 --- /dev/null +++ b/lib/rb/lib/thrift/transport/memory_buffer_transport.rb @@ -0,0 +1,93 @@ +# encoding: ascii-8bit +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +module Thrift + class MemoryBufferTransport < BaseTransport + GARBAGE_BUFFER_SIZE = 4*(2**10) # 4kB + + # If you pass a string to this, you should #dup that string + # unless you want it to be modified by #read and #write + #-- + # this behavior is no longer required. If you wish to change it + # go ahead, just make sure the specs pass + def initialize(buffer = nil) + @buf = buffer || '' + @index = 0 + end + + def open? + return true + end + + def open + end + + def close + end + + def peek + @index < @buf.size + end + + # this method does not use the passed object directly but copies it + def reset_buffer(new_buf = '') + @buf.replace new_buf + @index = 0 + end + + def available + @buf.length - @index + end + + def read(len) + data = @buf.slice(@index, len) + @index += len + @index = @buf.size if @index > @buf.size + if @index >= GARBAGE_BUFFER_SIZE + @buf = @buf.slice(@index..-1) + @index = 0 + end + data + end + + def write(wbuf) + @buf << wbuf + end + + def flush + end + + def inspect_buffer + out = [] + for idx in 0...(@buf.size) + # if idx != 0 + # out << " " + # end + + if idx == @index + out << ">" + end + + out << @buf[idx].to_s(16) + end + out.join(" ") + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/transport/server_socket.rb b/lib/rb/lib/thrift/transport/server_socket.rb new file mode 100644 index 00000000..7feb9ab0 --- /dev/null +++ b/lib/rb/lib/thrift/transport/server_socket.rb @@ -0,0 +1,63 @@ +# encoding: ascii-8bit +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'socket' + +module Thrift + class ServerSocket < BaseServerTransport + # call-seq: initialize(host = nil, port) + def initialize(host_or_port, port = nil) + if port + @host = host_or_port + @port = port + else + @host = nil + @port = host_or_port + end + @handle = nil + end + + attr_reader :handle + + def listen + @handle = TCPServer.new(@host, @port) + end + + def accept + unless @handle.nil? + sock = @handle.accept + trans = Socket.new + trans.handle = sock + trans + end + end + + def close + @handle.close unless @handle.nil? or @handle.closed? + @handle = nil + end + + def closed? + @handle.nil? or @handle.closed? + end + + alias to_io handle + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/transport/socket.rb b/lib/rb/lib/thrift/transport/socket.rb new file mode 100644 index 00000000..06c937e5 --- /dev/null +++ b/lib/rb/lib/thrift/transport/socket.rb @@ -0,0 +1,136 @@ +# encoding: ascii-8bit +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'socket' + +module Thrift + class Socket < BaseTransport + def initialize(host='localhost', port=9090, timeout=nil) + @host = host + @port = port + @timeout = timeout + @desc = "#{host}:#{port}" + @handle = nil + end + + attr_accessor :handle, :timeout + + def open + begin + addrinfo = ::Socket::getaddrinfo(@host, @port).first + @handle = ::Socket.new(addrinfo[4], ::Socket::SOCK_STREAM, 0) + sockaddr = ::Socket.sockaddr_in(addrinfo[1], addrinfo[3]) + begin + @handle.connect_nonblock(sockaddr) + rescue Errno::EINPROGRESS + unless IO.select(nil, [ @handle ], nil, @timeout) + raise TransportException.new(TransportException::NOT_OPEN, "Connection timeout to #{@desc}") + end + begin + @handle.connect_nonblock(sockaddr) + rescue Errno::EISCONN + end + end + @handle + rescue StandardError => e + raise TransportException.new(TransportException::NOT_OPEN, "Could not connect to #{@desc}: #{e}") + end + end + + def open? + !@handle.nil? and !@handle.closed? + end + + def write(str) + raise IOError, "closed stream" unless open? + begin + if @timeout.nil? or @timeout == 0 + @handle.write(str) + else + len = 0 + start = Time.now + while Time.now - start < @timeout + rd, wr, = IO.select(nil, [@handle], nil, @timeout) + if wr and not wr.empty? + len += @handle.write_nonblock(str[len..-1]) + break if len >= str.length + end + end + if len < str.length + raise TransportException.new(TransportException::TIMED_OUT, "Socket: Timed out writing #{str.length} bytes to #{@desc}") + else + len + end + end + rescue TransportException => e + # pass this on + raise e + rescue StandardError => e + @handle.close + @handle = nil + raise TransportException.new(TransportException::NOT_OPEN, e.message) + end + end + + def read(sz) + raise IOError, "closed stream" unless open? + + begin + if @timeout.nil? or @timeout == 0 + data = @handle.readpartial(sz) + else + # it's possible to interrupt select for something other than the timeout + # so we need to ensure we've waited long enough + start = Time.now + rd = nil # scoping + loop do + rd, = IO.select([@handle], nil, nil, @timeout) + break if (rd and not rd.empty?) or Time.now - start >= @timeout + end + if rd.nil? or rd.empty? + raise TransportException.new(TransportException::TIMED_OUT, "Socket: Timed out reading #{sz} bytes from #{@desc}") + else + data = @handle.readpartial(sz) + end + end + rescue TransportException => e + # don't let this get caught by the StandardError handler + raise e + rescue StandardError => e + @handle.close unless @handle.closed? + @handle = nil + raise TransportException.new(TransportException::NOT_OPEN, e.message) + end + if (data.nil? or data.length == 0) + raise TransportException.new(TransportException::UNKNOWN, "Socket: Could not read #{sz} bytes from #{@desc}") + end + data + end + + def close + @handle.close unless @handle.nil? or @handle.closed? + @handle = nil + end + + def to_io + @handle + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/transport/unix_server_socket.rb b/lib/rb/lib/thrift/transport/unix_server_socket.rb new file mode 100644 index 00000000..a135d25f --- /dev/null +++ b/lib/rb/lib/thrift/transport/unix_server_socket.rb @@ -0,0 +1,60 @@ +# encoding: ascii-8bit +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'socket' + +module Thrift + class UNIXServerSocket < BaseServerTransport + def initialize(path) + @path = path + @handle = nil + end + + attr_accessor :handle + + def listen + @handle = ::UNIXServer.new(@path) + end + + def accept + unless @handle.nil? + sock = @handle.accept + trans = UNIXSocket.new(nil) + trans.handle = sock + trans + end + end + + def close + if @handle + @handle.close unless @handle.closed? + @handle = nil + # UNIXServer doesn't delete the socket file, so we have to do it ourselves + File.delete(@path) + end + end + + def closed? + @handle.nil? or @handle.closed? + end + + alias to_io handle + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/transport/unix_socket.rb b/lib/rb/lib/thrift/transport/unix_socket.rb new file mode 100644 index 00000000..8f692e4c --- /dev/null +++ b/lib/rb/lib/thrift/transport/unix_socket.rb @@ -0,0 +1,40 @@ +# encoding: ascii-8bit +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'socket' + +module Thrift + class UNIXSocket < Socket + def initialize(path, timeout=nil) + @path = path + @timeout = timeout + @desc = @path # for read()'s error + @handle = nil + end + + def open + begin + @handle = ::UNIXSocket.new(@path) + rescue StandardError + raise TransportException.new(TransportException::NOT_OPEN, "Could not open UNIX socket at #{@path}") + end + end + end +end \ No newline at end of file diff --git a/lib/rb/lib/thrift/types.rb b/lib/rb/lib/thrift/types.rb new file mode 100644 index 00000000..20e4ca2c --- /dev/null +++ b/lib/rb/lib/thrift/types.rb @@ -0,0 +1,101 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'set' + +module Thrift + module Types + STOP = 0 + VOID = 1 + BOOL = 2 + BYTE = 3 + DOUBLE = 4 + I16 = 6 + I32 = 8 + I64 = 10 + STRING = 11 + STRUCT = 12 + MAP = 13 + SET = 14 + LIST = 15 + end + + class << self + attr_accessor :type_checking + end + + class TypeError < Exception + end + + def self.check_type(value, field, name, skip_nil=true) + return if value.nil? and skip_nil + klasses = case field[:type] + when Types::VOID + NilClass + when Types::BOOL + [TrueClass, FalseClass] + when Types::BYTE, Types::I16, Types::I32, Types::I64 + Integer + when Types::DOUBLE + Float + when Types::STRING + String + when Types::STRUCT + Struct + when Types::MAP + Hash + when Types::SET + Set + when Types::LIST + Array + end + valid = klasses && [*klasses].any? { |klass| klass === value } + raise TypeError, "Expected #{type_name(field[:type])}, received #{value.class} for field #{name}" unless valid + # check elements now + case field[:type] + when Types::MAP + value.each_pair do |k,v| + check_type(k, field[:key], "#{name}.key", false) + check_type(v, field[:value], "#{name}.value", false) + end + when Types::SET, Types::LIST + value.each do |el| + check_type(el, field[:element], "#{name}.element", false) + end + when Types::STRUCT + raise TypeError, "Expected #{field[:class]}, received #{value.class} for field #{name}" unless field[:class] == value.class + end + end + + def self.type_name(type) + Types.constants.each do |const| + return "Types::#{const}" if Types.const_get(const) == type + end + nil + end + + module MessageTypes + CALL = 1 + REPLY = 2 + EXCEPTION = 3 + ONEWAY = 4 + end +end + +Thrift.type_checking = false if Thrift.type_checking.nil? diff --git a/lib/rb/script/proto_benchmark.rb b/lib/rb/script/proto_benchmark.rb new file mode 100644 index 00000000..bb49e2e4 --- /dev/null +++ b/lib/rb/script/proto_benchmark.rb @@ -0,0 +1,121 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + "/../spec/spec_helper.rb" + +require "benchmark" +# require "ruby-prof" + +obj = Fixtures::COMPACT_PROTOCOL_TEST_STRUCT + +HOW_MANY = 1_000 + +binser = Thrift::Serializer.new +bin_data = binser.serialize(obj) +bindeser = Thrift::Deserializer.new +accel_bin_ser = Thrift::Serializer.new(Thrift::BinaryProtocolAcceleratedFactory.new) +accel_bin_deser = Thrift::Deserializer.new(Thrift::BinaryProtocolAcceleratedFactory.new) + +compact_ser = Thrift::Serializer.new(Thrift::CompactProtocolFactory.new) +compact_data = compact_ser.serialize(obj) +compact_deser = Thrift::Deserializer.new(Thrift::CompactProtocolFactory.new) + +Benchmark.bm(60) do |reporter| + reporter.report("binary protocol, write") do + HOW_MANY.times do + binser.serialize(obj) + end + end + + reporter.report("accelerated binary protocol, write") do + HOW_MANY.times do + accel_bin_ser.serialize(obj) + end + end + + reporter.report("compact protocol, write") do + # RubyProf.start + HOW_MANY.times do + compact_ser.serialize(obj) + end + # result = RubyProf.stop + # printer = RubyProf::GraphHtmlPrinter.new(result) + # file = File.open("profile.html", "w+") + # printer.print(file, 0) + # file.close + end + + reporter.report("binary protocol, read") do + HOW_MANY.times do + bindeser.deserialize(obj, bin_data) + end + end + + reporter.report("accelerated binary protocol, read") do + HOW_MANY.times do + accel_bin_deser.deserialize(obj, bin_data) + end + end + + reporter.report("compact protocol, read") do + HOW_MANY.times do + compact_deser.deserialize(obj, compact_data) + end + end + + + # f = File.new("/tmp/testfile", "w") + # proto = Thrift::BinaryProtocolAccelerated.new(Thrift::IOStreamTransport.new(Thrift::MemoryBufferTransport.new, f)) + # reporter.report("accelerated binary protocol, write (to disk)") do + # HOW_MANY.times do + # obj.write(proto) + # end + # f.flush + # end + # f.close + # + # f = File.new("/tmp/testfile", "r") + # proto = Thrift::BinaryProtocolAccelerated.new(Thrift::IOStreamTransport.new(f, Thrift::MemoryBufferTransport.new)) + # reporter.report("accelerated binary protocol, read (from disk)") do + # HOW_MANY.times do + # obj.read(proto) + # end + # end + # f.close + # + # f = File.new("/tmp/testfile", "w") + # reporter.report("compact protocol, write (to disk)") do + # proto = Thrift::CompactProtocol.new(Thrift::IOStreamTransport.new(Thrift::MemoryBufferTransport.new, f)) + # HOW_MANY.times do + # obj.write(proto) + # end + # f.flush + # end + # f.close + # + # f = File.new("/tmp/testfile", "r") + # reporter.report("compact protocol, read (from disk)") do + # proto = Thrift::CompactProtocol.new(Thrift::IOStreamTransport.new(f, Thrift::MemoryBufferTransport.new)) + # HOW_MANY.times do + # obj.read(proto) + # end + # end + # f.close + +end diff --git a/lib/rb/script/read_struct.rb b/lib/rb/script/read_struct.rb new file mode 100644 index 00000000..831fcec9 --- /dev/null +++ b/lib/rb/script/read_struct.rb @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require "spec/spec_helper" + +path, factory_class = ARGV + +factory = eval(factory_class).new + +deser = Thrift::Deserializer.new(factory) + +cpts = CompactProtoTestStruct.new +CompactProtoTestStruct.constants.each do |const| + cpts.instance_variable_set("@#{const}", nil) +end + +data = File.read(path) + +deser.deserialize(cpts, data) + +if cpts == Fixtures::COMPACT_PROTOCOL_TEST_STRUCT + puts "Object verified successfully!" +else + puts "Object failed verification! Expected #{Fixtures::COMPACT_PROTOCOL_TEST_STRUCT.inspect} but got #{cpts.inspect}" + + puts cpts.differences(Fixtures::COMPACT_PROTOCOL_TEST_STRUCT) +end diff --git a/lib/rb/script/write_struct.rb b/lib/rb/script/write_struct.rb new file mode 100644 index 00000000..da142197 --- /dev/null +++ b/lib/rb/script/write_struct.rb @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require "spec/spec_helper" + +path, factory_class = ARGV + +factory = eval(factory_class).new + +ser = Thrift::Serializer.new(factory) + +File.open(path, "w") do |file| + file.write(ser.serialize(Fixtures::COMPACT_PROTOCOL_TEST_STRUCT)) +end \ No newline at end of file diff --git a/lib/rb/setup.rb b/lib/rb/setup.rb new file mode 100644 index 00000000..9f0c8267 --- /dev/null +++ b/lib/rb/setup.rb @@ -0,0 +1,1585 @@ +# +# setup.rb +# +# Copyright (c) 2000-2005 Minero Aoki +# +# This program is free software. +# You can distribute/modify this program under the terms of +# the GNU LGPL, Lesser General Public License version 2.1. +# + +unless Enumerable.method_defined?(:map) # Ruby 1.4.6 + module Enumerable + alias map collect + end +end + +unless File.respond_to?(:read) # Ruby 1.6 + def File.read(fname) + open(fname) {|f| + return f.read + } + end +end + +unless Errno.const_defined?(:ENOTEMPTY) # Windows? + module Errno + class ENOTEMPTY + # We do not raise this exception, implementation is not needed. + end + end +end + +def File.binread(fname) + open(fname, 'rb') {|f| + return f.read + } +end + +# for corrupted Windows' stat(2) +def File.dir?(path) + File.directory?((path[-1,1] == '/') ? path : path + '/') +end + + +class ConfigTable + + include Enumerable + + def initialize(rbconfig) + @rbconfig = rbconfig + @items = [] + @table = {} + # options + @install_prefix = nil + @config_opt = nil + @verbose = true + @no_harm = false + end + + attr_accessor :install_prefix + attr_accessor :config_opt + + attr_writer :verbose + + def verbose? + @verbose + end + + attr_writer :no_harm + + def no_harm? + @no_harm + end + + def [](key) + lookup(key).resolve(self) + end + + def []=(key, val) + lookup(key).set val + end + + def names + @items.map {|i| i.name } + end + + def each(&block) + @items.each(&block) + end + + def key?(name) + @table.key?(name) + end + + def lookup(name) + @table[name] or setup_rb_error "no such config item: #{name}" + end + + def add(item) + @items.push item + @table[item.name] = item + end + + def remove(name) + item = lookup(name) + @items.delete_if {|i| i.name == name } + @table.delete_if {|name, i| i.name == name } + item + end + + def load_script(path, inst = nil) + if File.file?(path) + MetaConfigEnvironment.new(self, inst).instance_eval File.read(path), path + end + end + + def savefile + '.config' + end + + def load_savefile + begin + File.foreach(savefile()) do |line| + k, v = *line.split(/=/, 2) + self[k] = v.strip + end + rescue Errno::ENOENT + setup_rb_error $!.message + "\n#{File.basename($0)} config first" + end + end + + def save + @items.each {|i| i.value } + File.open(savefile(), 'w') {|f| + @items.each do |i| + f.printf "%s=%s\n", i.name, i.value if i.value? and i.value + end + } + end + + def load_standard_entries + standard_entries(@rbconfig).each do |ent| + add ent + end + end + + def standard_entries(rbconfig) + c = rbconfig + + rubypath = File.join(c['bindir'], c['ruby_install_name'] + c['EXEEXT']) + + major = c['MAJOR'].to_i + minor = c['MINOR'].to_i + teeny = c['TEENY'].to_i + version = "#{major}.#{minor}" + + # ruby ver. >= 1.4.4? + newpath_p = ((major >= 2) or + ((major == 1) and + ((minor >= 5) or + ((minor == 4) and (teeny >= 4))))) + + if c['rubylibdir'] + # V > 1.6.3 + libruby = "#{c['prefix']}/lib/ruby" + librubyver = c['rubylibdir'] + librubyverarch = c['archdir'] + siteruby = c['sitedir'] + siterubyver = c['sitelibdir'] + siterubyverarch = c['sitearchdir'] + elsif newpath_p + # 1.4.4 <= V <= 1.6.3 + libruby = "#{c['prefix']}/lib/ruby" + librubyver = "#{c['prefix']}/lib/ruby/#{version}" + librubyverarch = "#{c['prefix']}/lib/ruby/#{version}/#{c['arch']}" + siteruby = c['sitedir'] + siterubyver = "$siteruby/#{version}" + siterubyverarch = "$siterubyver/#{c['arch']}" + else + # V < 1.4.4 + libruby = "#{c['prefix']}/lib/ruby" + librubyver = "#{c['prefix']}/lib/ruby/#{version}" + librubyverarch = "#{c['prefix']}/lib/ruby/#{version}/#{c['arch']}" + siteruby = "#{c['prefix']}/lib/ruby/#{version}/site_ruby" + siterubyver = siteruby + siterubyverarch = "$siterubyver/#{c['arch']}" + end + parameterize = lambda {|path| + path.sub(/\A#{Regexp.quote(c['prefix'])}/, '$prefix') + } + + if arg = c['configure_args'].split.detect {|arg| /--with-make-prog=/ =~ arg } + makeprog = arg.sub(/'/, '').split(/=/, 2)[1] + else + makeprog = 'make' + end + + [ + ExecItem.new('installdirs', 'std/site/home', + 'std: install under libruby; site: install under site_ruby; home: install under $HOME')\ + {|val, table| + case val + when 'std' + table['rbdir'] = '$librubyver' + table['sodir'] = '$librubyverarch' + when 'site' + table['rbdir'] = '$siterubyver' + table['sodir'] = '$siterubyverarch' + when 'home' + setup_rb_error '$HOME was not set' unless ENV['HOME'] + table['prefix'] = ENV['HOME'] + table['rbdir'] = '$libdir/ruby' + table['sodir'] = '$libdir/ruby' + end + }, + PathItem.new('prefix', 'path', c['prefix'], + 'path prefix of target environment'), + PathItem.new('bindir', 'path', parameterize.call(c['bindir']), + 'the directory for commands'), + PathItem.new('libdir', 'path', parameterize.call(c['libdir']), + 'the directory for libraries'), + PathItem.new('datadir', 'path', parameterize.call(c['datadir']), + 'the directory for shared data'), + PathItem.new('mandir', 'path', parameterize.call(c['mandir']), + 'the directory for man pages'), + PathItem.new('sysconfdir', 'path', parameterize.call(c['sysconfdir']), + 'the directory for system configuration files'), + PathItem.new('localstatedir', 'path', parameterize.call(c['localstatedir']), + 'the directory for local state data'), + PathItem.new('libruby', 'path', libruby, + 'the directory for ruby libraries'), + PathItem.new('librubyver', 'path', librubyver, + 'the directory for standard ruby libraries'), + PathItem.new('librubyverarch', 'path', librubyverarch, + 'the directory for standard ruby extensions'), + PathItem.new('siteruby', 'path', siteruby, + 'the directory for version-independent aux ruby libraries'), + PathItem.new('siterubyver', 'path', siterubyver, + 'the directory for aux ruby libraries'), + PathItem.new('siterubyverarch', 'path', siterubyverarch, + 'the directory for aux ruby binaries'), + PathItem.new('rbdir', 'path', '$siterubyver', + 'the directory for ruby scripts'), + PathItem.new('sodir', 'path', '$siterubyverarch', + 'the directory for ruby extentions'), + PathItem.new('rubypath', 'path', rubypath, + 'the path to set to #! line'), + ProgramItem.new('rubyprog', 'name', rubypath, + 'the ruby program using for installation'), + ProgramItem.new('makeprog', 'name', makeprog, + 'the make program to compile ruby extentions'), + SelectItem.new('shebang', 'all/ruby/never', 'ruby', + 'shebang line (#!) editing mode'), + BoolItem.new('without-ext', 'yes/no', 'no', + 'does not compile/install ruby extentions') + ] + end + private :standard_entries + + def load_multipackage_entries + multipackage_entries().each do |ent| + add ent + end + end + + def multipackage_entries + [ + PackageSelectionItem.new('with', 'name,name...', '', 'ALL', + 'package names that you want to install'), + PackageSelectionItem.new('without', 'name,name...', '', 'NONE', + 'package names that you do not want to install') + ] + end + private :multipackage_entries + + ALIASES = { + 'std-ruby' => 'librubyver', + 'stdruby' => 'librubyver', + 'rubylibdir' => 'librubyver', + 'archdir' => 'librubyverarch', + 'site-ruby-common' => 'siteruby', # For backward compatibility + 'site-ruby' => 'siterubyver', # For backward compatibility + 'bin-dir' => 'bindir', + 'bin-dir' => 'bindir', + 'rb-dir' => 'rbdir', + 'so-dir' => 'sodir', + 'data-dir' => 'datadir', + 'ruby-path' => 'rubypath', + 'ruby-prog' => 'rubyprog', + 'ruby' => 'rubyprog', + 'make-prog' => 'makeprog', + 'make' => 'makeprog' + } + + def fixup + ALIASES.each do |ali, name| + @table[ali] = @table[name] + end + @items.freeze + @table.freeze + @options_re = /\A--(#{@table.keys.join('|')})(?:=(.*))?\z/ + end + + def parse_opt(opt) + m = @options_re.match(opt) or setup_rb_error "config: unknown option #{opt}" + m.to_a[1,2] + end + + def dllext + @rbconfig['DLEXT'] + end + + def value_config?(name) + lookup(name).value? + end + + class Item + def initialize(name, template, default, desc) + @name = name.freeze + @template = template + @value = default + @default = default + @description = desc + end + + attr_reader :name + attr_reader :description + + attr_accessor :default + alias help_default default + + def help_opt + "--#{@name}=#{@template}" + end + + def value? + true + end + + def value + @value + end + + def resolve(table) + @value.gsub(%r<\$([^/]+)>) { table[$1] } + end + + def set(val) + @value = check(val) + end + + private + + def check(val) + setup_rb_error "config: --#{name} requires argument" unless val + val + end + end + + class BoolItem < Item + def config_type + 'bool' + end + + def help_opt + "--#{@name}" + end + + private + + def check(val) + return 'yes' unless val + case val + when /\Ay(es)?\z/i, /\At(rue)?\z/i then 'yes' + when /\An(o)?\z/i, /\Af(alse)\z/i then 'no' + else + setup_rb_error "config: --#{@name} accepts only yes/no for argument" + end + end + end + + class PathItem < Item + def config_type + 'path' + end + + private + + def check(path) + setup_rb_error "config: --#{@name} requires argument" unless path + path[0,1] == '$' ? path : File.expand_path(path) + end + end + + class ProgramItem < Item + def config_type + 'program' + end + end + + class SelectItem < Item + def initialize(name, selection, default, desc) + super + @ok = selection.split('/') + end + + def config_type + 'select' + end + + private + + def check(val) + unless @ok.include?(val.strip) + setup_rb_error "config: use --#{@name}=#{@template} (#{val})" + end + val.strip + end + end + + class ExecItem < Item + def initialize(name, selection, desc, &block) + super name, selection, nil, desc + @ok = selection.split('/') + @action = block + end + + def config_type + 'exec' + end + + def value? + false + end + + def resolve(table) + setup_rb_error "$#{name()} wrongly used as option value" + end + + undef set + + def evaluate(val, table) + v = val.strip.downcase + unless @ok.include?(v) + setup_rb_error "invalid option --#{@name}=#{val} (use #{@template})" + end + @action.call v, table + end + end + + class PackageSelectionItem < Item + def initialize(name, template, default, help_default, desc) + super name, template, default, desc + @help_default = help_default + end + + attr_reader :help_default + + def config_type + 'package' + end + + private + + def check(val) + unless File.dir?("packages/#{val}") + setup_rb_error "config: no such package: #{val}" + end + val + end + end + + class MetaConfigEnvironment + def initialize(config, installer) + @config = config + @installer = installer + end + + def config_names + @config.names + end + + def config?(name) + @config.key?(name) + end + + def bool_config?(name) + @config.lookup(name).config_type == 'bool' + end + + def path_config?(name) + @config.lookup(name).config_type == 'path' + end + + def value_config?(name) + @config.lookup(name).config_type != 'exec' + end + + def add_config(item) + @config.add item + end + + def add_bool_config(name, default, desc) + @config.add BoolItem.new(name, 'yes/no', default ? 'yes' : 'no', desc) + end + + def add_path_config(name, default, desc) + @config.add PathItem.new(name, 'path', default, desc) + end + + def set_config_default(name, default) + @config.lookup(name).default = default + end + + def remove_config(name) + @config.remove(name) + end + + # For only multipackage + def packages + raise '[setup.rb fatal] multi-package metaconfig API packages() called for single-package; contact application package vendor' unless @installer + @installer.packages + end + + # For only multipackage + def declare_packages(list) + raise '[setup.rb fatal] multi-package metaconfig API declare_packages() called for single-package; contact application package vendor' unless @installer + @installer.packages = list + end + end + +end # class ConfigTable + + +# This module requires: #verbose?, #no_harm? +module FileOperations + + def mkdir_p(dirname, prefix = nil) + dirname = prefix + File.expand_path(dirname) if prefix + $stderr.puts "mkdir -p #{dirname}" if verbose? + return if no_harm? + + # Does not check '/', it's too abnormal. + dirs = File.expand_path(dirname).split(%r<(?=/)>) + if /\A[a-z]:\z/i =~ dirs[0] + disk = dirs.shift + dirs[0] = disk + dirs[0] + end + dirs.each_index do |idx| + path = dirs[0..idx].join('') + Dir.mkdir path unless File.dir?(path) + end + end + + def rm_f(path) + $stderr.puts "rm -f #{path}" if verbose? + return if no_harm? + force_remove_file path + end + + def rm_rf(path) + $stderr.puts "rm -rf #{path}" if verbose? + return if no_harm? + remove_tree path + end + + def remove_tree(path) + if File.symlink?(path) + remove_file path + elsif File.dir?(path) + remove_tree0 path + else + force_remove_file path + end + end + + def remove_tree0(path) + Dir.foreach(path) do |ent| + next if ent == '.' + next if ent == '..' + entpath = "#{path}/#{ent}" + if File.symlink?(entpath) + remove_file entpath + elsif File.dir?(entpath) + remove_tree0 entpath + else + force_remove_file entpath + end + end + begin + Dir.rmdir path + rescue Errno::ENOTEMPTY + # directory may not be empty + end + end + + def move_file(src, dest) + force_remove_file dest + begin + File.rename src, dest + rescue + File.open(dest, 'wb') {|f| + f.write File.binread(src) + } + File.chmod File.stat(src).mode, dest + File.unlink src + end + end + + def force_remove_file(path) + begin + remove_file path + rescue + end + end + + def remove_file(path) + File.chmod 0777, path + File.unlink path + end + + def install(from, dest, mode, prefix = nil) + $stderr.puts "install #{from} #{dest}" if verbose? + return if no_harm? + + realdest = prefix ? prefix + File.expand_path(dest) : dest + realdest = File.join(realdest, File.basename(from)) if File.dir?(realdest) + str = File.binread(from) + if diff?(str, realdest) + verbose_off { + rm_f realdest if File.exist?(realdest) + } + File.open(realdest, 'wb') {|f| + f.write str + } + File.chmod mode, realdest + + File.open("#{objdir_root()}/InstalledFiles", 'a') {|f| + if prefix + f.puts realdest.sub(prefix, '') + else + f.puts realdest + end + } + end + end + + def diff?(new_content, path) + return true unless File.exist?(path) + new_content != File.binread(path) + end + + def command(*args) + $stderr.puts args.join(' ') if verbose? + system(*args) or raise RuntimeError, + "system(#{args.map{|a| a.inspect }.join(' ')}) failed" + end + + def ruby(*args) + command config('rubyprog'), *args + end + + def make(task = nil) + command(*[config('makeprog'), task].compact) + end + + def extdir?(dir) + File.exist?("#{dir}/MANIFEST") or File.exist?("#{dir}/extconf.rb") + end + + def files_of(dir) + Dir.open(dir) {|d| + return d.select {|ent| File.file?("#{dir}/#{ent}") } + } + end + + DIR_REJECT = %w( . .. CVS SCCS RCS CVS.adm .svn ) + + def directories_of(dir) + Dir.open(dir) {|d| + return d.select {|ent| File.dir?("#{dir}/#{ent}") } - DIR_REJECT + } + end + +end + + +# This module requires: #srcdir_root, #objdir_root, #relpath +module HookScriptAPI + + def get_config(key) + @config[key] + end + + alias config get_config + + # obsolete: use metaconfig to change configuration + def set_config(key, val) + @config[key] = val + end + + # + # srcdir/objdir (works only in the package directory) + # + + def curr_srcdir + "#{srcdir_root()}/#{relpath()}" + end + + def curr_objdir + "#{objdir_root()}/#{relpath()}" + end + + def srcfile(path) + "#{curr_srcdir()}/#{path}" + end + + def srcexist?(path) + File.exist?(srcfile(path)) + end + + def srcdirectory?(path) + File.dir?(srcfile(path)) + end + + def srcfile?(path) + File.file?(srcfile(path)) + end + + def srcentries(path = '.') + Dir.open("#{curr_srcdir()}/#{path}") {|d| + return d.to_a - %w(. ..) + } + end + + def srcfiles(path = '.') + srcentries(path).select {|fname| + File.file?(File.join(curr_srcdir(), path, fname)) + } + end + + def srcdirectories(path = '.') + srcentries(path).select {|fname| + File.dir?(File.join(curr_srcdir(), path, fname)) + } + end + +end + + +class ToplevelInstaller + + Version = '3.4.1' + Copyright = 'Copyright (c) 2000-2005 Minero Aoki' + + TASKS = [ + [ 'all', 'do config, setup, then install' ], + [ 'config', 'saves your configurations' ], + [ 'show', 'shows current configuration' ], + [ 'setup', 'compiles ruby extentions and others' ], + [ 'install', 'installs files' ], + [ 'test', 'run all tests in test/' ], + [ 'clean', "does `make clean' for each extention" ], + [ 'distclean',"does `make distclean' for each extention" ] + ] + + def ToplevelInstaller.invoke + config = ConfigTable.new(load_rbconfig()) + config.load_standard_entries + config.load_multipackage_entries if multipackage? + config.fixup + klass = (multipackage?() ? ToplevelInstallerMulti : ToplevelInstaller) + klass.new(File.dirname($0), config).invoke + end + + def ToplevelInstaller.multipackage? + File.dir?(File.dirname($0) + '/packages') + end + + def ToplevelInstaller.load_rbconfig + if arg = ARGV.detect {|arg| /\A--rbconfig=/ =~ arg } + ARGV.delete(arg) + load File.expand_path(arg.split(/=/, 2)[1]) + $".push 'rbconfig.rb' + else + require 'rbconfig' + end + ::Config::CONFIG + end + + def initialize(ardir_root, config) + @ardir = File.expand_path(ardir_root) + @config = config + # cache + @valid_task_re = nil + end + + def config(key) + @config[key] + end + + def inspect + "#<#{self.class} #{__id__()}>" + end + + def invoke + run_metaconfigs + case task = parsearg_global() + when nil, 'all' + parsearg_config + init_installers + exec_config + exec_setup + exec_install + else + case task + when 'config', 'test' + ; + when 'clean', 'distclean' + @config.load_savefile if File.exist?(@config.savefile) + else + @config.load_savefile + end + __send__ "parsearg_#{task}" + init_installers + __send__ "exec_#{task}" + end + end + + def run_metaconfigs + @config.load_script "#{@ardir}/metaconfig" + end + + def init_installers + @installer = Installer.new(@config, @ardir, File.expand_path('.')) + end + + # + # Hook Script API bases + # + + def srcdir_root + @ardir + end + + def objdir_root + '.' + end + + def relpath + '.' + end + + # + # Option Parsing + # + + def parsearg_global + while arg = ARGV.shift + case arg + when /\A\w+\z/ + setup_rb_error "invalid task: #{arg}" unless valid_task?(arg) + return arg + when '-q', '--quiet' + @config.verbose = false + when '--verbose' + @config.verbose = true + when '--help' + print_usage $stdout + exit 0 + when '--version' + puts "#{File.basename($0)} version #{Version}" + exit 0 + when '--copyright' + puts Copyright + exit 0 + else + setup_rb_error "unknown global option '#{arg}'" + end + end + nil + end + + def valid_task?(t) + valid_task_re() =~ t + end + + def valid_task_re + @valid_task_re ||= /\A(?:#{TASKS.map {|task,desc| task }.join('|')})\z/ + end + + def parsearg_no_options + unless ARGV.empty? + task = caller(0).first.slice(%r<`parsearg_(\w+)'>, 1) + setup_rb_error "#{task}: unknown options: #{ARGV.join(' ')}" + end + end + + alias parsearg_show parsearg_no_options + alias parsearg_setup parsearg_no_options + alias parsearg_test parsearg_no_options + alias parsearg_clean parsearg_no_options + alias parsearg_distclean parsearg_no_options + + def parsearg_config + evalopt = [] + set = [] + @config.config_opt = [] + while i = ARGV.shift + if /\A--?\z/ =~ i + @config.config_opt = ARGV.dup + break + end + name, value = *@config.parse_opt(i) + if @config.value_config?(name) + @config[name] = value + else + evalopt.push [name, value] + end + set.push name + end + evalopt.each do |name, value| + @config.lookup(name).evaluate value, @config + end + # Check if configuration is valid + set.each do |n| + @config[n] if @config.value_config?(n) + end + end + + def parsearg_install + @config.no_harm = false + @config.install_prefix = '' + while a = ARGV.shift + case a + when '--no-harm' + @config.no_harm = true + when /\A--prefix=/ + path = a.split(/=/, 2)[1] + path = File.expand_path(path) unless path[0,1] == '/' + @config.install_prefix = path + else + setup_rb_error "install: unknown option #{a}" + end + end + end + + def print_usage(out) + out.puts 'Typical Installation Procedure:' + out.puts " $ ruby #{File.basename $0} config" + out.puts " $ ruby #{File.basename $0} setup" + out.puts " # ruby #{File.basename $0} install (may require root privilege)" + out.puts + out.puts 'Detailed Usage:' + out.puts " ruby #{File.basename $0} " + out.puts " ruby #{File.basename $0} [] []" + + fmt = " %-24s %s\n" + out.puts + out.puts 'Global options:' + out.printf fmt, '-q,--quiet', 'suppress message outputs' + out.printf fmt, ' --verbose', 'output messages verbosely' + out.printf fmt, ' --help', 'print this message' + out.printf fmt, ' --version', 'print version and quit' + out.printf fmt, ' --copyright', 'print copyright and quit' + out.puts + out.puts 'Tasks:' + TASKS.each do |name, desc| + out.printf fmt, name, desc + end + + fmt = " %-24s %s [%s]\n" + out.puts + out.puts 'Options for CONFIG or ALL:' + @config.each do |item| + out.printf fmt, item.help_opt, item.description, item.help_default + end + out.printf fmt, '--rbconfig=path', 'rbconfig.rb to load',"running ruby's" + out.puts + out.puts 'Options for INSTALL:' + out.printf fmt, '--no-harm', 'only display what to do if given', 'off' + out.printf fmt, '--prefix=path', 'install path prefix', '' + out.puts + end + + # + # Task Handlers + # + + def exec_config + @installer.exec_config + @config.save # must be final + end + + def exec_setup + @installer.exec_setup + end + + def exec_install + @installer.exec_install + end + + def exec_test + @installer.exec_test + end + + def exec_show + @config.each do |i| + printf "%-20s %s\n", i.name, i.value if i.value? + end + end + + def exec_clean + @installer.exec_clean + end + + def exec_distclean + @installer.exec_distclean + end + +end # class ToplevelInstaller + + +class ToplevelInstallerMulti < ToplevelInstaller + + include FileOperations + + def initialize(ardir_root, config) + super + @packages = directories_of("#{@ardir}/packages") + raise 'no package exists' if @packages.empty? + @root_installer = Installer.new(@config, @ardir, File.expand_path('.')) + end + + def run_metaconfigs + @config.load_script "#{@ardir}/metaconfig", self + @packages.each do |name| + @config.load_script "#{@ardir}/packages/#{name}/metaconfig" + end + end + + attr_reader :packages + + def packages=(list) + raise 'package list is empty' if list.empty? + list.each do |name| + raise "directory packages/#{name} does not exist"\ + unless File.dir?("#{@ardir}/packages/#{name}") + end + @packages = list + end + + def init_installers + @installers = {} + @packages.each do |pack| + @installers[pack] = Installer.new(@config, + "#{@ardir}/packages/#{pack}", + "packages/#{pack}") + end + with = extract_selection(config('with')) + without = extract_selection(config('without')) + @selected = @installers.keys.select {|name| + (with.empty? or with.include?(name)) \ + and not without.include?(name) + } + end + + def extract_selection(list) + a = list.split(/,/) + a.each do |name| + setup_rb_error "no such package: #{name}" unless @installers.key?(name) + end + a + end + + def print_usage(f) + super + f.puts 'Inluded packages:' + f.puts ' ' + @packages.sort.join(' ') + f.puts + end + + # + # Task Handlers + # + + def exec_config + run_hook 'pre-config' + each_selected_installers {|inst| inst.exec_config } + run_hook 'post-config' + @config.save # must be final + end + + def exec_setup + run_hook 'pre-setup' + each_selected_installers {|inst| inst.exec_setup } + run_hook 'post-setup' + end + + def exec_install + run_hook 'pre-install' + each_selected_installers {|inst| inst.exec_install } + run_hook 'post-install' + end + + def exec_test + run_hook 'pre-test' + each_selected_installers {|inst| inst.exec_test } + run_hook 'post-test' + end + + def exec_clean + rm_f @config.savefile + run_hook 'pre-clean' + each_selected_installers {|inst| inst.exec_clean } + run_hook 'post-clean' + end + + def exec_distclean + rm_f @config.savefile + run_hook 'pre-distclean' + each_selected_installers {|inst| inst.exec_distclean } + run_hook 'post-distclean' + end + + # + # lib + # + + def each_selected_installers + Dir.mkdir 'packages' unless File.dir?('packages') + @selected.each do |pack| + $stderr.puts "Processing the package `#{pack}' ..." if verbose? + Dir.mkdir "packages/#{pack}" unless File.dir?("packages/#{pack}") + Dir.chdir "packages/#{pack}" + yield @installers[pack] + Dir.chdir '../..' + end + end + + def run_hook(id) + @root_installer.run_hook id + end + + # module FileOperations requires this + def verbose? + @config.verbose? + end + + # module FileOperations requires this + def no_harm? + @config.no_harm? + end + +end # class ToplevelInstallerMulti + + +class Installer + + FILETYPES = %w( bin lib ext data conf man ) + + include FileOperations + include HookScriptAPI + + def initialize(config, srcroot, objroot) + @config = config + @srcdir = File.expand_path(srcroot) + @objdir = File.expand_path(objroot) + @currdir = '.' + end + + def inspect + "#<#{self.class} #{File.basename(@srcdir)}>" + end + + def noop(rel) + end + + # + # Hook Script API base methods + # + + def srcdir_root + @srcdir + end + + def objdir_root + @objdir + end + + def relpath + @currdir + end + + # + # Config Access + # + + # module FileOperations requires this + def verbose? + @config.verbose? + end + + # module FileOperations requires this + def no_harm? + @config.no_harm? + end + + def verbose_off + begin + save, @config.verbose = @config.verbose?, false + yield + ensure + @config.verbose = save + end + end + + # + # TASK config + # + + def exec_config + exec_task_traverse 'config' + end + + alias config_dir_bin noop + alias config_dir_lib noop + + def config_dir_ext(rel) + extconf if extdir?(curr_srcdir()) + end + + alias config_dir_data noop + alias config_dir_conf noop + alias config_dir_man noop + + def extconf + ruby "#{curr_srcdir()}/extconf.rb", *@config.config_opt + end + + # + # TASK setup + # + + def exec_setup + exec_task_traverse 'setup' + end + + def setup_dir_bin(rel) + files_of(curr_srcdir()).each do |fname| + update_shebang_line "#{curr_srcdir()}/#{fname}" + end + end + + alias setup_dir_lib noop + + def setup_dir_ext(rel) + make if extdir?(curr_srcdir()) + end + + alias setup_dir_data noop + alias setup_dir_conf noop + alias setup_dir_man noop + + def update_shebang_line(path) + return if no_harm? + return if config('shebang') == 'never' + old = Shebang.load(path) + if old + $stderr.puts "warning: #{path}: Shebang line includes too many args. It is not portable and your program may not work." if old.args.size > 1 + new = new_shebang(old) + return if new.to_s == old.to_s + else + return unless config('shebang') == 'all' + new = Shebang.new(config('rubypath')) + end + $stderr.puts "updating shebang: #{File.basename(path)}" if verbose? + open_atomic_writer(path) {|output| + File.open(path, 'rb') {|f| + f.gets if old # discard + output.puts new.to_s + output.print f.read + } + } + end + + def new_shebang(old) + if /\Aruby/ =~ File.basename(old.cmd) + Shebang.new(config('rubypath'), old.args) + elsif File.basename(old.cmd) == 'env' and old.args.first == 'ruby' + Shebang.new(config('rubypath'), old.args[1..-1]) + else + return old unless config('shebang') == 'all' + Shebang.new(config('rubypath')) + end + end + + def open_atomic_writer(path, &block) + tmpfile = File.basename(path) + '.tmp' + begin + File.open(tmpfile, 'wb', &block) + File.rename tmpfile, File.basename(path) + ensure + File.unlink tmpfile if File.exist?(tmpfile) + end + end + + class Shebang + def Shebang.load(path) + line = nil + File.open(path) {|f| + line = f.gets + } + return nil unless /\A#!/ =~ line + parse(line) + end + + def Shebang.parse(line) + cmd, *args = *line.strip.sub(/\A\#!/, '').split(' ') + new(cmd, args) + end + + def initialize(cmd, args = []) + @cmd = cmd + @args = args + end + + attr_reader :cmd + attr_reader :args + + def to_s + "#! #{@cmd}" + (@args.empty? ? '' : " #{@args.join(' ')}") + end + end + + # + # TASK install + # + + def exec_install + rm_f 'InstalledFiles' + exec_task_traverse 'install' + end + + def install_dir_bin(rel) + install_files targetfiles(), "#{config('bindir')}/#{rel}", 0755 + end + + def install_dir_lib(rel) + install_files libfiles(), "#{config('rbdir')}/#{rel}", 0644 + end + + def install_dir_ext(rel) + return unless extdir?(curr_srcdir()) + install_files rubyextentions('.'), + "#{config('sodir')}/#{File.dirname(rel)}", + 0555 + end + + def install_dir_data(rel) + install_files targetfiles(), "#{config('datadir')}/#{rel}", 0644 + end + + def install_dir_conf(rel) + # FIXME: should not remove current config files + # (rename previous file to .old/.org) + install_files targetfiles(), "#{config('sysconfdir')}/#{rel}", 0644 + end + + def install_dir_man(rel) + install_files targetfiles(), "#{config('mandir')}/#{rel}", 0644 + end + + def install_files(list, dest, mode) + mkdir_p dest, @config.install_prefix + list.each do |fname| + install fname, dest, mode, @config.install_prefix + end + end + + def libfiles + glob_reject(%w(*.y *.output), targetfiles()) + end + + def rubyextentions(dir) + ents = glob_select("*.#{@config.dllext}", targetfiles()) + if ents.empty? + setup_rb_error "no ruby extention exists: 'ruby #{$0} setup' first" + end + ents + end + + def targetfiles + mapdir(existfiles() - hookfiles()) + end + + def mapdir(ents) + ents.map {|ent| + if File.exist?(ent) + then ent # objdir + else "#{curr_srcdir()}/#{ent}" # srcdir + end + } + end + + # picked up many entries from cvs-1.11.1/src/ignore.c + JUNK_FILES = %w( + core RCSLOG tags TAGS .make.state + .nse_depinfo #* .#* cvslog.* ,* .del-* *.olb + *~ *.old *.bak *.BAK *.orig *.rej _$* *$ + + *.org *.in .* + ) + + def existfiles + glob_reject(JUNK_FILES, (files_of(curr_srcdir()) | files_of('.'))) + end + + def hookfiles + %w( pre-%s post-%s pre-%s.rb post-%s.rb ).map {|fmt| + %w( config setup install clean ).map {|t| sprintf(fmt, t) } + }.flatten + end + + def glob_select(pat, ents) + re = globs2re([pat]) + ents.select {|ent| re =~ ent } + end + + def glob_reject(pats, ents) + re = globs2re(pats) + ents.reject {|ent| re =~ ent } + end + + GLOB2REGEX = { + '.' => '\.', + '$' => '\$', + '#' => '\#', + '*' => '.*' + } + + def globs2re(pats) + /\A(?:#{ + pats.map {|pat| pat.gsub(/[\.\$\#\*]/) {|ch| GLOB2REGEX[ch] } }.join('|') + })\z/ + end + + # + # TASK test + # + + TESTDIR = 'test' + + def exec_test + unless File.directory?('test') + $stderr.puts 'no test in this package' if verbose? + return + end + $stderr.puts 'Running tests...' if verbose? + begin + require 'test/unit' + rescue LoadError + setup_rb_error 'test/unit cannot loaded. You need Ruby 1.8 or later to invoke this task.' + end + runner = Test::Unit::AutoRunner.new(true) + runner.to_run << TESTDIR + runner.run + end + + # + # TASK clean + # + + def exec_clean + exec_task_traverse 'clean' + rm_f @config.savefile + rm_f 'InstalledFiles' + end + + alias clean_dir_bin noop + alias clean_dir_lib noop + alias clean_dir_data noop + alias clean_dir_conf noop + alias clean_dir_man noop + + def clean_dir_ext(rel) + return unless extdir?(curr_srcdir()) + make 'clean' if File.file?('Makefile') + end + + # + # TASK distclean + # + + def exec_distclean + exec_task_traverse 'distclean' + rm_f @config.savefile + rm_f 'InstalledFiles' + end + + alias distclean_dir_bin noop + alias distclean_dir_lib noop + + def distclean_dir_ext(rel) + return unless extdir?(curr_srcdir()) + make 'distclean' if File.file?('Makefile') + end + + alias distclean_dir_data noop + alias distclean_dir_conf noop + alias distclean_dir_man noop + + # + # Traversing + # + + def exec_task_traverse(task) + run_hook "pre-#{task}" + FILETYPES.each do |type| + if type == 'ext' and config('without-ext') == 'yes' + $stderr.puts 'skipping ext/* by user option' if verbose? + next + end + traverse task, type, "#{task}_dir_#{type}" + end + run_hook "post-#{task}" + end + + def traverse(task, rel, mid) + dive_into(rel) { + run_hook "pre-#{task}" + __send__ mid, rel.sub(%r[\A.*?(?:/|\z)], '') + directories_of(curr_srcdir()).each do |d| + traverse task, "#{rel}/#{d}", mid + end + run_hook "post-#{task}" + } + end + + def dive_into(rel) + return unless File.dir?("#{@srcdir}/#{rel}") + + dir = File.basename(rel) + Dir.mkdir dir unless File.dir?(dir) + prevdir = Dir.pwd + Dir.chdir dir + $stderr.puts '---> ' + rel if verbose? + @currdir = rel + yield + Dir.chdir prevdir + $stderr.puts '<--- ' + rel if verbose? + @currdir = File.dirname(rel) + end + + def run_hook(id) + path = [ "#{curr_srcdir()}/#{id}", + "#{curr_srcdir()}/#{id}.rb" ].detect {|cand| File.file?(cand) } + return unless path + begin + instance_eval File.read(path), path, 1 + rescue + raise if $DEBUG + setup_rb_error "hook #{path} failed:\n" + $!.message + end + end + +end # class Installer + + +class SetupError < StandardError; end + +def setup_rb_error(msg) + raise SetupError, msg +end + +if $0 == __FILE__ + begin + ToplevelInstaller.invoke + rescue SetupError + raise if $DEBUG + $stderr.puts $!.message + $stderr.puts "Try 'ruby #{$0} --help' for detailed usage." + exit 1 + end +end diff --git a/lib/rb/spec/ThriftSpec.thrift b/lib/rb/spec/ThriftSpec.thrift new file mode 100644 index 00000000..fe5a8aae --- /dev/null +++ b/lib/rb/spec/ThriftSpec.thrift @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +namespace rb SpecNamespace + +struct Hello { + 1: string greeting = "hello world" +} + +struct Foo { + 1: i32 simple = 53, + 2: string words = "words", + 3: Hello hello = {'greeting' : "hello, world!"}, + 4: list ints = [1, 2, 2, 3], + 5: map> complex, + 6: set shorts = [5, 17, 239], + 7: optional string opt_string +} + +struct BoolStruct { + 1: bool yesno = 1 +} + +struct SimpleList { + 1: list bools, + 2: list bytes, + 3: list i16s, + 4: list i32s, + 5: list i64s, + 6: list doubles, + 7: list strings, + 8: list> maps, + 9: list> lists, + 10: list> sets, + 11: list hellos +} + +exception Xception { + 1: string message, + 2: i32 code = 1 +} + +service NonblockingService { + Hello greeting(1:bool english) + bool block() + oneway void unblock(1:i32 n) + oneway void shutdown() + void sleep(1:double seconds) +} diff --git a/lib/rb/spec/base_protocol_spec.rb b/lib/rb/spec/base_protocol_spec.rb new file mode 100644 index 00000000..efb16d8c --- /dev/null +++ b/lib/rb/spec/base_protocol_spec.rb @@ -0,0 +1,160 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' + +class ThriftBaseProtocolSpec < Spec::ExampleGroup + include Thrift + + before(:each) do + @trans = mock("MockTransport") + @prot = BaseProtocol.new(@trans) + end + + describe BaseProtocol do + # most of the methods are stubs, so we can ignore them + + it "should make trans accessible" do + @prot.trans.should eql(@trans) + end + + it "should write out a field nicely" do + @prot.should_receive(:write_field_begin).with('field', 'type', 'fid').ordered + @prot.should_receive(:write_type).with('type', 'value').ordered + @prot.should_receive(:write_field_end).ordered + @prot.write_field('field', 'type', 'fid', 'value') + end + + it "should write out the different types" do + @prot.should_receive(:write_bool).with('bool').ordered + @prot.should_receive(:write_byte).with('byte').ordered + @prot.should_receive(:write_double).with('double').ordered + @prot.should_receive(:write_i16).with('i16').ordered + @prot.should_receive(:write_i32).with('i32').ordered + @prot.should_receive(:write_i64).with('i64').ordered + @prot.should_receive(:write_string).with('string').ordered + struct = mock('Struct') + struct.should_receive(:write).with(@prot).ordered + @prot.write_type(Types::BOOL, 'bool') + @prot.write_type(Types::BYTE, 'byte') + @prot.write_type(Types::DOUBLE, 'double') + @prot.write_type(Types::I16, 'i16') + @prot.write_type(Types::I32, 'i32') + @prot.write_type(Types::I64, 'i64') + @prot.write_type(Types::STRING, 'string') + @prot.write_type(Types::STRUCT, struct) + # all other types are not implemented + [Types::STOP, Types::VOID, Types::MAP, Types::SET, Types::LIST].each do |type| + lambda { @prot.write_type(type, type.to_s) }.should raise_error(NotImplementedError) + end + end + + it "should read the different types" do + @prot.should_receive(:read_bool).ordered + @prot.should_receive(:read_byte).ordered + @prot.should_receive(:read_i16).ordered + @prot.should_receive(:read_i32).ordered + @prot.should_receive(:read_i64).ordered + @prot.should_receive(:read_double).ordered + @prot.should_receive(:read_string).ordered + @prot.read_type(Types::BOOL) + @prot.read_type(Types::BYTE) + @prot.read_type(Types::I16) + @prot.read_type(Types::I32) + @prot.read_type(Types::I64) + @prot.read_type(Types::DOUBLE) + @prot.read_type(Types::STRING) + # all other types are not implemented + [Types::STOP, Types::VOID, Types::MAP, Types::SET, Types::LIST].each do |type| + lambda { @prot.read_type(type) }.should raise_error(NotImplementedError) + end + end + + it "should skip the basic types" do + @prot.should_receive(:read_bool).ordered + @prot.should_receive(:read_byte).ordered + @prot.should_receive(:read_i16).ordered + @prot.should_receive(:read_i32).ordered + @prot.should_receive(:read_i64).ordered + @prot.should_receive(:read_double).ordered + @prot.should_receive(:read_string).ordered + @prot.skip(Types::BOOL) + @prot.skip(Types::BYTE) + @prot.skip(Types::I16) + @prot.skip(Types::I32) + @prot.skip(Types::I64) + @prot.skip(Types::DOUBLE) + @prot.skip(Types::STRING) + @prot.skip(Types::STOP) # should do absolutely nothing + end + + it "should skip structs" do + real_skip = @prot.method(:skip) + @prot.should_receive(:read_struct_begin).ordered + @prot.should_receive(:read_field_begin).exactly(4).times.and_return( + ['field 1', Types::STRING, 1], + ['field 2', Types::I32, 2], + ['field 3', Types::MAP, 3], + [nil, Types::STOP, 0] + ) + @prot.should_receive(:read_field_end).exactly(3).times + @prot.should_receive(:read_string).exactly(3).times + @prot.should_receive(:read_i32).ordered + @prot.should_receive(:read_map_begin).ordered.and_return([Types::STRING, Types::STRING, 1]) + # @prot.should_receive(:read_string).exactly(2).times + @prot.should_receive(:read_map_end).ordered + @prot.should_receive(:read_struct_end).ordered + real_skip.call(Types::STRUCT) + end + + it "should skip maps" do + real_skip = @prot.method(:skip) + @prot.should_receive(:read_map_begin).ordered.and_return([Types::STRING, Types::STRUCT, 1]) + @prot.should_receive(:read_string).ordered + @prot.should_receive(:read_struct_begin).ordered.and_return(["some_struct"]) + @prot.should_receive(:read_field_begin).ordered.and_return([nil, Types::STOP, nil]); + @prot.should_receive(:read_struct_end).ordered + @prot.should_receive(:read_map_end).ordered + real_skip.call(Types::MAP) + end + + it "should skip sets" do + real_skip = @prot.method(:skip) + @prot.should_receive(:read_set_begin).ordered.and_return([Types::I64, 9]) + @prot.should_receive(:read_i64).ordered.exactly(9).times + @prot.should_receive(:read_set_end) + real_skip.call(Types::SET) + end + + it "should skip lists" do + real_skip = @prot.method(:skip) + @prot.should_receive(:read_list_begin).ordered.and_return([Types::DOUBLE, 11]) + @prot.should_receive(:read_double).ordered.exactly(11).times + @prot.should_receive(:read_list_end) + real_skip.call(Types::LIST) + end + end + + describe BaseProtocolFactory do + it "should raise NotImplementedError" do + # returning nil since Protocol is just an abstract class + lambda {BaseProtocolFactory.new.get_protocol(mock("MockTransport"))}.should raise_error(NotImplementedError) + end + end +end diff --git a/lib/rb/spec/base_transport_spec.rb b/lib/rb/spec/base_transport_spec.rb new file mode 100644 index 00000000..71897759 --- /dev/null +++ b/lib/rb/spec/base_transport_spec.rb @@ -0,0 +1,344 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' + +class ThriftBaseTransportSpec < Spec::ExampleGroup + include Thrift + + describe TransportException do + it "should make type accessible" do + exc = TransportException.new(TransportException::ALREADY_OPEN, "msg") + exc.type.should == TransportException::ALREADY_OPEN + exc.message.should == "msg" + end + end + + describe BaseTransport do + it "should read the specified size" do + transport = BaseTransport.new + transport.should_receive(:read).with(40).ordered.and_return("10 letters") + transport.should_receive(:read).with(30).ordered.and_return("fifteen letters") + transport.should_receive(:read).with(15).ordered.and_return("more characters") + transport.read_all(40).should == "10 lettersfifteen lettersmore characters" + end + + it "should stub out the rest of the methods" do + # can't test for stubbiness, so just make sure they're defined + [:open?, :open, :close, :read, :write, :flush].each do |sym| + BaseTransport.method_defined?(sym).should be_true + end + end + + it "should alias << to write" do + BaseTransport.instance_method(:<<).should == BaseTransport.instance_method(:write) + end + end + + describe BaseServerTransport do + it "should stub out its methods" do + [:listen, :accept, :close].each do |sym| + BaseServerTransport.method_defined?(sym).should be_true + end + end + end + + describe BaseTransportFactory do + it "should return the transport it's given" do + transport = mock("Transport") + BaseTransportFactory.new.get_transport(transport).should eql(transport) + end + end + + describe BufferedTransport do + it "should pass through everything but write/flush/read" do + trans = mock("Transport") + trans.should_receive(:open?).ordered.and_return("+ open?") + trans.should_receive(:open).ordered.and_return("+ open") + trans.should_receive(:flush).ordered # from the close + trans.should_receive(:close).ordered.and_return("+ close") + btrans = BufferedTransport.new(trans) + btrans.open?.should == "+ open?" + btrans.open.should == "+ open" + btrans.close.should == "+ close" + end + + it "should buffer reads in chunks of #{BufferedTransport::DEFAULT_BUFFER}" do + trans = mock("Transport") + trans.should_receive(:read).with(BufferedTransport::DEFAULT_BUFFER).and_return("lorum ipsum dolor emet") + btrans = BufferedTransport.new(trans) + btrans.read(6).should == "lorum " + btrans.read(6).should == "ipsum " + btrans.read(6).should == "dolor " + btrans.read(6).should == "emet" + end + + it "should buffer writes and send them on flush" do + trans = mock("Transport") + btrans = BufferedTransport.new(trans) + btrans.write("one/") + btrans.write("two/") + btrans.write("three/") + trans.should_receive(:write).with("one/two/three/").ordered + trans.should_receive(:flush).ordered + btrans.flush + end + + it "should only send buffered data once" do + trans = mock("Transport") + btrans = BufferedTransport.new(trans) + btrans.write("one/") + btrans.write("two/") + btrans.write("three/") + trans.should_receive(:write).with("one/two/three/") + trans.stub!(:flush) + btrans.flush + # Nothing to flush with no data + btrans.flush + end + + it "should flush on close" do + trans = mock("Transport") + trans.should_receive(:close) + btrans = BufferedTransport.new(trans) + btrans.should_receive(:flush) + btrans.close + end + + it "should not write to socket if there's no data" do + trans = mock("Transport") + trans.should_receive(:flush) + btrans = BufferedTransport.new(trans) + btrans.flush + end + end + + describe BufferedTransportFactory do + it "should wrap the given transport in a BufferedTransport" do + trans = mock("Transport") + btrans = mock("BufferedTransport") + BufferedTransport.should_receive(:new).with(trans).and_return(btrans) + BufferedTransportFactory.new.get_transport(trans).should == btrans + end + end + + describe FramedTransport do + before(:each) do + @trans = mock("Transport") + end + + it "should pass through open?/open/close" do + ftrans = FramedTransport.new(@trans) + @trans.should_receive(:open?).ordered.and_return("+ open?") + @trans.should_receive(:open).ordered.and_return("+ open") + @trans.should_receive(:close).ordered.and_return("+ close") + ftrans.open?.should == "+ open?" + ftrans.open.should == "+ open" + ftrans.close.should == "+ close" + end + + it "should pass through read when read is turned off" do + ftrans = FramedTransport.new(@trans, false, true) + @trans.should_receive(:read).with(17).ordered.and_return("+ read") + ftrans.read(17).should == "+ read" + end + + it "should pass through write/flush when write is turned off" do + ftrans = FramedTransport.new(@trans, true, false) + @trans.should_receive(:write).with("foo").ordered.and_return("+ write") + @trans.should_receive(:flush).ordered.and_return("+ flush") + ftrans.write("foo").should == "+ write" + ftrans.flush.should == "+ flush" + end + + it "should return a full frame if asked for >= the frame's length" do + frame = "this is a frame" + @trans.should_receive(:read_all).with(4).and_return("\000\000\000\017") + @trans.should_receive(:read_all).with(frame.length).and_return(frame) + FramedTransport.new(@trans).read(frame.length + 10).should == frame + end + + it "should return slices of the frame when asked for < the frame's length" do + frame = "this is a frame" + @trans.should_receive(:read_all).with(4).and_return("\000\000\000\017") + @trans.should_receive(:read_all).with(frame.length).and_return(frame) + ftrans = FramedTransport.new(@trans) + ftrans.read(4).should == "this" + ftrans.read(4).should == " is " + ftrans.read(16).should == "a frame" + end + + it "should return nothing if asked for <= 0" do + FramedTransport.new(@trans).read(-2).should == "" + end + + it "should pull a new frame when the first is exhausted" do + frame = "this is a frame" + frame2 = "yet another frame" + @trans.should_receive(:read_all).with(4).and_return("\000\000\000\017", "\000\000\000\021") + @trans.should_receive(:read_all).with(frame.length).and_return(frame) + @trans.should_receive(:read_all).with(frame2.length).and_return(frame2) + ftrans = FramedTransport.new(@trans) + ftrans.read(4).should == "this" + ftrans.read(8).should == " is a fr" + ftrans.read(6).should == "ame" + ftrans.read(4).should == "yet " + ftrans.read(16).should == "another frame" + end + + it "should buffer writes" do + ftrans = FramedTransport.new(@trans) + @trans.should_not_receive(:write) + ftrans.write("foo") + ftrans.write("bar") + ftrans.write("this is a frame") + end + + it "should write slices of the buffer" do + ftrans = FramedTransport.new(@trans) + ftrans.write("foobar", 3) + ftrans.write("barfoo", 1) + @trans.stub!(:flush) + @trans.should_receive(:write).with("\000\000\000\004foob") + ftrans.flush + end + + it "should flush frames with a 4-byte header" do + ftrans = FramedTransport.new(@trans) + @trans.should_receive(:write).with("\000\000\000\035one/two/three/this is a frame").ordered + @trans.should_receive(:flush).ordered + ftrans.write("one/") + ftrans.write("two/") + ftrans.write("three/") + ftrans.write("this is a frame") + ftrans.flush + end + + it "should not flush the same buffered data twice" do + ftrans = FramedTransport.new(@trans) + @trans.should_receive(:write).with("\000\000\000\007foo/bar") + @trans.stub!(:flush) + ftrans.write("foo") + ftrans.write("/bar") + ftrans.flush + @trans.should_receive(:write).with("\000\000\000\000") + ftrans.flush + end + end + + describe FramedTransportFactory do + it "should wrap the given transport in a FramedTransport" do + trans = mock("Transport") + FramedTransport.should_receive(:new).with(trans) + FramedTransportFactory.new.get_transport(trans) + end + end + + describe MemoryBufferTransport do + before(:each) do + @buffer = MemoryBufferTransport.new + end + + it "should accept a buffer on input and use it directly" do + s = "this is a test" + @buffer = MemoryBufferTransport.new(s) + @buffer.read(4).should == "this" + s.slice!(-4..-1) + @buffer.read(@buffer.available).should == " is a " + end + + it "should always remain open" do + @buffer.should be_open + @buffer.close + @buffer.should be_open + end + + it "should respond to peek and available" do + @buffer.write "some data" + @buffer.peek.should be_true + @buffer.available.should == 9 + @buffer.read(4) + @buffer.peek.should be_true + @buffer.available.should == 5 + @buffer.read(16) + @buffer.peek.should be_false + @buffer.available.should == 0 + end + + it "should be able to reset the buffer" do + @buffer.write "test data" + @buffer.reset_buffer("foobar") + @buffer.available.should == 6 + @buffer.read(10).should == "foobar" + @buffer.reset_buffer + @buffer.available.should == 0 + end + + it "should copy the given string whne resetting the buffer" do + s = "this is a test" + @buffer.reset_buffer(s) + @buffer.available.should == 14 + @buffer.read(10) + @buffer.available.should == 4 + s.should == "this is a test" + end + + it "should return from read what was given in write" do + @buffer.write "test data" + @buffer.read(4).should == "test" + @buffer.read(10).should == " data" + @buffer.read(10).should == "" + @buffer.write "foo" + @buffer.write " bar" + @buffer.read(10).should == "foo bar" + end + end + + describe IOStreamTransport do + before(:each) do + @input = mock("Input", :closed? => false) + @output = mock("Output", :closed? => false) + @trans = IOStreamTransport.new(@input, @output) + end + + it "should be open as long as both input or output are open" do + @trans.should be_open + @input.stub!(:closed?).and_return(true) + @trans.should be_open + @input.stub!(:closed?).and_return(false) + @output.stub!(:closed?).and_return(true) + @trans.should be_open + @input.stub!(:closed?).and_return(true) + @trans.should_not be_open + end + + it "should pass through read/write to input/output" do + @input.should_receive(:read).with(17).and_return("+ read") + @output.should_receive(:write).with("foobar").and_return("+ write") + @trans.read(17).should == "+ read" + @trans.write("foobar").should == "+ write" + end + + it "should close both input and output when closed" do + @input.should_receive(:close) + @output.should_receive(:close) + @trans.close + end + end +end diff --git a/lib/rb/spec/binary_protocol_accelerated_spec.rb b/lib/rb/spec/binary_protocol_accelerated_spec.rb new file mode 100644 index 00000000..0306cf5f --- /dev/null +++ b/lib/rb/spec/binary_protocol_accelerated_spec.rb @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' +require File.dirname(__FILE__) + '/binary_protocol_spec_shared' +require File.dirname(__FILE__) + '/gen-rb/thrift_spec_types' + +class ThriftBinaryProtocolAcceleratedSpec < Spec::ExampleGroup + include Thrift + + describe Thrift::BinaryProtocolAccelerated do + # since BinaryProtocolAccelerated should be directly equivalent to + # BinaryProtocol, we don't need any custom specs! + it_should_behave_like 'a binary protocol' + + def protocol_class + BinaryProtocolAccelerated + end + end + + describe BinaryProtocolAcceleratedFactory do + it "should create a BinaryProtocolAccelerated" do + BinaryProtocolAcceleratedFactory.new.get_protocol(mock("MockTransport")).should be_instance_of(BinaryProtocolAccelerated) + end + end +end diff --git a/lib/rb/spec/binary_protocol_spec.rb b/lib/rb/spec/binary_protocol_spec.rb new file mode 100644 index 00000000..0abccb89 --- /dev/null +++ b/lib/rb/spec/binary_protocol_spec.rb @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' +require File.dirname(__FILE__) + '/binary_protocol_spec_shared' + +class ThriftBinaryProtocolSpec < Spec::ExampleGroup + include Thrift + + describe BinaryProtocol do + it_should_behave_like 'a binary protocol' + + def protocol_class + BinaryProtocol + end + + it "should read a message header" do + @trans.should_receive(:read_all).exactly(2).times.and_return( + [protocol_class.const_get(:VERSION_1) | Thrift::MessageTypes::REPLY].pack('N'), + [42].pack('N') + ) + @prot.should_receive(:read_string).and_return('testMessage') + @prot.read_message_begin.should == ['testMessage', Thrift::MessageTypes::REPLY, 42] + end + + it "should raise an exception if the message header has the wrong version" do + @prot.should_receive(:read_i32).and_return(-1) + lambda { @prot.read_message_begin }.should raise_error(Thrift::ProtocolException, 'Missing version identifier') do |e| + e.type == Thrift::ProtocolException::BAD_VERSION + end + end + + it "should raise an exception if the message header does not exist and strict_read is enabled" do + @prot.should_receive(:read_i32).and_return(42) + @prot.should_receive(:strict_read).and_return(true) + lambda { @prot.read_message_begin }.should raise_error(Thrift::ProtocolException, 'No version identifier, old protocol client?') do |e| + e.type == Thrift::ProtocolException::BAD_VERSION + end + end + end + + describe BinaryProtocolFactory do + it "should create a BinaryProtocol" do + BinaryProtocolFactory.new.get_protocol(mock("MockTransport")).should be_instance_of(BinaryProtocol) + end + end +end diff --git a/lib/rb/spec/binary_protocol_spec_shared.rb b/lib/rb/spec/binary_protocol_spec_shared.rb new file mode 100644 index 00000000..c6608e01 --- /dev/null +++ b/lib/rb/spec/binary_protocol_spec_shared.rb @@ -0,0 +1,375 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' + +shared_examples_for 'a binary protocol' do + before(:each) do + @trans = Thrift::MemoryBufferTransport.new + @prot = protocol_class.new(@trans) + end + + it "should define the proper VERSION_1, VERSION_MASK AND TYPE_MASK" do + protocol_class.const_get(:VERSION_MASK).should == 0xffff0000 + protocol_class.const_get(:VERSION_1).should == 0x80010000 + protocol_class.const_get(:TYPE_MASK).should == 0x000000ff + end + + it "should make strict_read readable" do + @prot.strict_read.should eql(true) + end + + it "should make strict_write readable" do + @prot.strict_write.should eql(true) + end + + it "should write the message header" do + @prot.write_message_begin('testMessage', Thrift::MessageTypes::CALL, 17) + @trans.read(1000).should == [protocol_class.const_get(:VERSION_1) | Thrift::MessageTypes::CALL, "testMessage".size, "testMessage", 17].pack("NNa11N") + end + + it "should write the message header without version when writes are not strict" do + @prot = protocol_class.new(@trans, true, false) # no strict write + @prot.write_message_begin('testMessage', Thrift::MessageTypes::CALL, 17) + @trans.read(1000).should == "\000\000\000\vtestMessage\001\000\000\000\021" + end + + it "should write the message header with a version when writes are strict" do + @prot = protocol_class.new(@trans) # strict write + @prot.write_message_begin('testMessage', Thrift::MessageTypes::CALL, 17) + @trans.read(1000).should == "\200\001\000\001\000\000\000\vtestMessage\000\000\000\021" + end + + + # message footer is a noop + + it "should write the field header" do + @prot.write_field_begin('foo', Thrift::Types::DOUBLE, 3) + @trans.read(1000).should == [Thrift::Types::DOUBLE, 3].pack("cn") + end + + # field footer is a noop + + it "should write the STOP field" do + @prot.write_field_stop + @trans.read(1).should == "\000" + end + + it "should write the map header" do + @prot.write_map_begin(Thrift::Types::STRING, Thrift::Types::LIST, 17) + @trans.read(1000).should == [Thrift::Types::STRING, Thrift::Types::LIST, 17].pack("ccN"); + end + + # map footer is a noop + + it "should write the list header" do + @prot.write_list_begin(Thrift::Types::I16, 42) + @trans.read(1000).should == [Thrift::Types::I16, 42].pack("cN") + end + + # list footer is a noop + + it "should write the set header" do + @prot.write_set_begin(Thrift::Types::I16, 42) + @trans.read(1000).should == [Thrift::Types::I16, 42].pack("cN") + end + + it "should write a bool" do + @prot.write_bool(true) + @prot.write_bool(false) + @trans.read(1000).should == "\001\000" + end + + it "should treat a nil bool as false" do + @prot.write_bool(nil) + @trans.read(1).should == "\000" + end + + it "should write a byte" do + # byte is small enough, let's check -128..127 + (-128..127).each do |i| + @prot.write_byte(i) + @trans.read(1).should == [i].pack('c') + end + # handing it numbers out of signed range should clip + @trans.rspec_verify + (128..255).each do |i| + @prot.write_byte(i) + @trans.read(1).should == [i].pack('c') + end + # and lastly, a Bignum is going to error out + lambda { @prot.write_byte(2**65) }.should raise_error(RangeError) + end + + it "should error gracefully when trying to write a nil byte" do + lambda { @prot.write_byte(nil) }.should raise_error + end + + it "should write an i16" do + # try a random scattering of values + # include the signed i16 minimum/maximum + [-2**15, -1024, 17, 0, -10000, 1723, 2**15-1].each do |i| + @prot.write_i16(i) + end + # and try something out of signed range, it should clip + @prot.write_i16(2**15 + 5) + + @trans.read(1000).should == "\200\000\374\000\000\021\000\000\330\360\006\273\177\377\200\005" + + # a Bignum should error + # lambda { @prot.write_i16(2**65) }.should raise_error(RangeError) + end + + it "should error gracefully when trying to write a nil i16" do + lambda { @prot.write_i16(nil) }.should raise_error + end + + it "should write an i32" do + # try a random scattering of values + # include the signed i32 minimum/maximum + [-2**31, -123123, -2532, -3, 0, 2351235, 12331, 2**31-1].each do |i| + @prot.write_i32(i) + end + # try something out of signed range, it should clip + @trans.read(1000).should == "\200\000\000\000" + "\377\376\037\r" + "\377\377\366\034" + "\377\377\377\375" + "\000\000\000\000" + "\000#\340\203" + "\000\0000+" + "\177\377\377\377" + [2 ** 31 + 5, 2 ** 65 + 5].each do |i| + lambda { @prot.write_i32(i) }.should raise_error(RangeError) + end + end + + it "should error gracefully when trying to write a nil i32" do + lambda { @prot.write_i32(nil) }.should raise_error + end + + it "should write an i64" do + # try a random scattering of values + # try the signed i64 minimum/maximum + [-2**63, -12356123612323, -23512351, -234, 0, 1231, 2351236, 12361236213, 2**63-1].each do |i| + @prot.write_i64(i) + end + # try something out of signed range, it should clip + @trans.read(1000).should == ["\200\000\000\000\000\000\000\000", + "\377\377\364\303\035\244+]", + "\377\377\377\377\376\231:\341", + "\377\377\377\377\377\377\377\026", + "\000\000\000\000\000\000\000\000", + "\000\000\000\000\000\000\004\317", + "\000\000\000\000\000#\340\204", + "\000\000\000\002\340\311~\365", + "\177\377\377\377\377\377\377\377"].join("") + lambda { @prot.write_i64(2 ** 65 + 5) }.should raise_error(RangeError) + end + + it "should error gracefully when trying to write a nil i64" do + lambda { @prot.write_i64(nil) }.should raise_error + end + + it "should write a double" do + # try a random scattering of values, including min/max + values = [Float::MIN,-1231.15325, -123123.23, -23.23515123, 0, 12351.1325, 523.23, Float::MAX] + values.each do |f| + @prot.write_double(f) + @trans.read(1000).should == [f].pack("G") + end + end + + it "should error gracefully when trying to write a nil double" do + lambda { @prot.write_double(nil) }.should raise_error + end + + it "should write a string" do + str = "hello world" + @prot.write_string(str) + @trans.read(1000).should == [str.size].pack("N") + str + end + + it "should error gracefully when trying to write a nil string" do + lambda { @prot.write_string(nil) }.should raise_error + end + + it "should write the message header without version when writes are not strict" do + @prot = protocol_class.new(@trans, true, false) # no strict write + @prot.write_message_begin('testMessage', Thrift::MessageTypes::CALL, 17) + @trans.read(1000).should == "\000\000\000\vtestMessage\001\000\000\000\021" + end + + it "should write the message header with a version when writes are strict" do + @prot = protocol_class.new(@trans) # strict write + @prot.write_message_begin('testMessage', Thrift::MessageTypes::CALL, 17) + @trans.read(1000).should == "\200\001\000\001\000\000\000\vtestMessage\000\000\000\021" + end + + # message footer is a noop + + it "should read a field header" do + @trans.write([Thrift::Types::STRING, 3].pack("cn")) + @prot.read_field_begin.should == [nil, Thrift::Types::STRING, 3] + end + + # field footer is a noop + + it "should read a stop field" do + @trans.write([Thrift::Types::STOP].pack("c")); + @prot.read_field_begin.should == [nil, Thrift::Types::STOP, 0] + end + + it "should read a map header" do + @trans.write([Thrift::Types::DOUBLE, Thrift::Types::I64, 42].pack("ccN")) + @prot.read_map_begin.should == [Thrift::Types::DOUBLE, Thrift::Types::I64, 42] + end + + # map footer is a noop + + it "should read a list header" do + @trans.write([Thrift::Types::STRING, 17].pack("cN")) + @prot.read_list_begin.should == [Thrift::Types::STRING, 17] + end + + # list footer is a noop + + it "should read a set header" do + @trans.write([Thrift::Types::STRING, 17].pack("cN")) + @prot.read_set_begin.should == [Thrift::Types::STRING, 17] + end + + # set footer is a noop + + it "should read a bool" do + @trans.write("\001\000"); + @prot.read_bool.should == true + @prot.read_bool.should == false + end + + it "should read a byte" do + [-128, -57, -3, 0, 17, 24, 127].each do |i| + @trans.write([i].pack("c")) + @prot.read_byte.should == i + end + end + + it "should read an i16" do + # try a scattering of values, including min/max + [-2**15, -5237, -353, 0, 1527, 2234, 2**15-1].each do |i| + @trans.write([i].pack("n")); + @prot.read_i16.should == i + end + end + + it "should read an i32" do + # try a scattering of values, including min/max + [-2**31, -235125, -6236, 0, 2351, 123123, 2**31-1].each do |i| + @trans.write([i].pack("N")) + @prot.read_i32.should == i + end + end + + it "should read an i64" do + # try a scattering of values, including min/max + [-2**63, -123512312, -6346, 0, 32, 2346322323, 2**63-1].each do |i| + @trans.write([i >> 32, i & 0xFFFFFFFF].pack("NN")) + @prot.read_i64.should == i + end + end + + it "should read a double" do + # try a random scattering of values, including min/max + [Float::MIN, -231231.12351, -323.233513, 0, 123.2351235, 2351235.12351235, Float::MAX].each do |f| + @trans.write([f].pack("G")); + @prot.read_double.should == f + end + end + + it "should read a string" do + str = "hello world" + @trans.write([str.size].pack("N") + str) + @prot.read_string.should == str + end + + it "should perform a complete rpc with no args or return" do + srv_test( + proc {|client| client.send_voidMethod()}, + proc {|client| client.recv_voidMethod.should == nil} + ) + end + + it "should perform a complete rpc with a primitive return type" do + srv_test( + proc {|client| client.send_primitiveMethod()}, + proc {|client| client.recv_primitiveMethod.should == 1} + ) + end + + it "should perform a complete rpc with a struct return type" do + srv_test( + proc {|client| client.send_structMethod()}, + proc {|client| + result = client.recv_structMethod + result.set_byte_map = nil + result.map_byte_map = nil + result.should == Fixtures::COMPACT_PROTOCOL_TEST_STRUCT + } + ) + end + + def get_socket_connection + server = Thrift::ServerSocket.new("localhost", 9090) + server.listen + + clientside = Thrift::Socket.new("localhost", 9090) + clientside.open + serverside = server.accept + [clientside, serverside, server] + end + + def srv_test(firstblock, secondblock) + clientside, serverside, server = get_socket_connection + + clientproto = protocol_class.new(clientside) + serverproto = protocol_class.new(serverside) + + processor = Srv::Processor.new(SrvHandler.new) + + client = Srv::Client.new(clientproto, clientproto) + + # first block + firstblock.call(client) + + processor.process(serverproto, serverproto) + + # second block + secondblock.call(client) + ensure + clientside.close + serverside.close + server.close + end + + class SrvHandler + def voidMethod() + end + + def primitiveMethod + 1 + end + + def structMethod + Fixtures::COMPACT_PROTOCOL_TEST_STRUCT + end + end +end diff --git a/lib/rb/spec/client_spec.rb b/lib/rb/spec/client_spec.rb new file mode 100644 index 00000000..e707d816 --- /dev/null +++ b/lib/rb/spec/client_spec.rb @@ -0,0 +1,100 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' + +class ThriftClientSpec < Spec::ExampleGroup + include Thrift + + class ClientSpec + include Thrift::Client + end + + before(:each) do + @prot = mock("MockProtocol") + @client = ClientSpec.new(@prot) + end + + describe Client do + it "should re-use iprot for oprot if not otherwise specified" do + @client.instance_variable_get(:'@iprot').should eql(@prot) + @client.instance_variable_get(:'@oprot').should eql(@prot) + end + + it "should send a test message" do + @prot.should_receive(:write_message_begin).with('testMessage', MessageTypes::CALL, 0) + mock_args = mock('#') + mock_args.should_receive(:foo=).with('foo') + mock_args.should_receive(:bar=).with(42) + mock_args.should_receive(:write).with(@prot) + @prot.should_receive(:write_message_end) + @prot.should_receive(:trans) do + mock('trans').tee do |trans| + trans.should_receive(:flush) + end + end + klass = stub("TestMessage_args", :new => mock_args) + @client.send_message('testMessage', klass, :foo => 'foo', :bar => 42) + end + + it "should increment the sequence id when sending messages" do + pending "it seems sequence ids are completely ignored right now" do + @prot.should_receive(:write_message_begin).with('testMessage', MessageTypes::CALL, 0).ordered + @prot.should_receive(:write_message_begin).with('testMessage2', MessageTypes::CALL, 1).ordered + @prot.should_receive(:write_message_begin).with('testMessage3', MessageTypes::CALL, 2).ordered + @prot.stub!(:write_message_end) + @prot.stub!(:trans).and_return mock("trans").as_null_object + @client.send_message('testMessage', mock("args class").as_null_object) + @client.send_message('testMessage2', mock("args class").as_null_object) + @client.send_message('testMessage3', mock("args class").as_null_object) + end + end + + it "should receive a test message" do + @prot.should_receive(:read_message_begin).and_return [nil, MessageTypes::CALL, 0] + @prot.should_receive(:read_message_end) + mock_klass = mock("#") + mock_klass.should_receive(:read).with(@prot) + @client.receive_message(stub("MockClass", :new => mock_klass)) + end + + it "should handle received exceptions" do + @prot.should_receive(:read_message_begin).and_return [nil, MessageTypes::EXCEPTION, 0] + @prot.should_receive(:read_message_end) + ApplicationException.should_receive(:new).and_return do + StandardError.new.tee do |mock_exc| + mock_exc.should_receive(:read).with(@prot) + end + end + lambda { @client.receive_message(nil) }.should raise_error(StandardError) + end + + it "should close the transport if an error occurs while sending a message" do + @prot.stub!(:write_message_begin) + @prot.should_not_receive(:write_message_end) + mock_args = mock("#") + mock_args.should_receive(:write).with(@prot).and_raise(StandardError) + trans = mock("MockTransport") + @prot.stub!(:trans).and_return(trans) + trans.should_receive(:close) + klass = mock("TestMessage_args", :new => mock_args) + lambda { @client.send_message("testMessage", klass) }.should raise_error(StandardError) + end + end +end diff --git a/lib/rb/spec/compact_protocol_spec.rb b/lib/rb/spec/compact_protocol_spec.rb new file mode 100644 index 00000000..b9a79810 --- /dev/null +++ b/lib/rb/spec/compact_protocol_spec.rb @@ -0,0 +1,117 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' + +describe Thrift::CompactProtocol do + TESTS = { + :byte => (-127..127).to_a, + :i16 => (0..14).map {|shift| [1 << shift, -(1 << shift)]}.flatten.sort, + :i32 => (0..30).map {|shift| [1 << shift, -(1 << shift)]}.flatten.sort, + :i64 => (0..62).map {|shift| [1 << shift, -(1 << shift)]}.flatten.sort, + :string => ["", "1", "short", "fourteen123456", "fifteen12345678", "1" * 127, "1" * 3000], + :binary => ["", "\001", "\001" * 5, "\001" * 14, "\001" * 15, "\001" * 127, "\001" * 3000], + :double => [0.0, 1.0, -1.0, 1.1, -1.1, 10000000.1, 1.0/0.0, -1.0/0.0], + :bool => [true, false] + } + + it "should encode and decode naked primitives correctly" do + TESTS.each_pair do |primitive_type, test_values| + test_values.each do |value| + # puts "testing #{value}" if primitive_type == :i64 + trans = Thrift::MemoryBufferTransport.new + proto = Thrift::CompactProtocol.new(trans) + + proto.send(writer(primitive_type), value) + # puts "buf: #{trans.inspect_buffer}" if primitive_type == :i64 + read_back = proto.send(reader(primitive_type)) + read_back.should == value + end + end + end + + it "should encode and decode primitives in fields correctly" do + TESTS.each_pair do |primitive_type, test_values| + final_primitive_type = primitive_type == :binary ? :string : primitive_type + thrift_type = Thrift::Types.const_get(final_primitive_type.to_s.upcase) + # puts primitive_type + test_values.each do |value| + trans = Thrift::MemoryBufferTransport.new + proto = Thrift::CompactProtocol.new(trans) + + proto.write_field_begin(nil, thrift_type, 15) + proto.send(writer(primitive_type), value) + proto.write_field_end + + proto = Thrift::CompactProtocol.new(trans) + name, type, id = proto.read_field_begin + type.should == thrift_type + id.should == 15 + read_back = proto.send(reader(primitive_type)) + read_back.should == value + proto.read_field_end + end + end + end + + it "should encode and decode a monster struct correctly" do + trans = Thrift::MemoryBufferTransport.new + proto = Thrift::CompactProtocol.new(trans) + + struct = CompactProtoTestStruct.new + # sets and maps don't hash well... not sure what to do here. + struct.write(proto) + + struct2 = CompactProtoTestStruct.new + struct2.read(proto) + struct2.should == struct + end + + it "should make method calls correctly" do + client_out_trans = Thrift::MemoryBufferTransport.new + client_out_proto = Thrift::CompactProtocol.new(client_out_trans) + + client_in_trans = Thrift::MemoryBufferTransport.new + client_in_proto = Thrift::CompactProtocol.new(client_in_trans) + + processor = Srv::Processor.new(JankyHandler.new) + + client = Srv::Client.new(client_in_proto, client_out_proto) + client.send_Janky(1) + # puts client_out_trans.inspect_buffer + processor.process(client_out_proto, client_in_proto) + client.recv_Janky.should == 2 + end + + class JankyHandler + def Janky(i32arg) + i32arg * 2 + end + end + + def writer(sym) + sym = sym == :binary ? :string : sym + "write_#{sym.to_s}" + end + + def reader(sym) + sym = sym == :binary ? :string : sym + "read_#{sym.to_s}" + end +end \ No newline at end of file diff --git a/lib/rb/spec/exception_spec.rb b/lib/rb/spec/exception_spec.rb new file mode 100644 index 00000000..fc321378 --- /dev/null +++ b/lib/rb/spec/exception_spec.rb @@ -0,0 +1,142 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' + +class ThriftExceptionSpec < Spec::ExampleGroup + include Thrift + + describe Exception do + it "should have an accessible message" do + e = Exception.new("test message") + e.message.should == "test message" + end + end + + describe ApplicationException do + it "should inherit from Thrift::Exception" do + ApplicationException.superclass.should == Exception + end + + it "should have an accessible type and message" do + e = ApplicationException.new + e.type.should == ApplicationException::UNKNOWN + e.message.should be_nil + e = ApplicationException.new(ApplicationException::UNKNOWN_METHOD, "test message") + e.type.should == ApplicationException::UNKNOWN_METHOD + e.message.should == "test message" + end + + it "should read a struct off of a protocol" do + prot = mock("MockProtocol") + prot.should_receive(:read_struct_begin).ordered + prot.should_receive(:read_field_begin).exactly(3).times.and_return( + ["message", Types::STRING, 1], + ["type", Types::I32, 2], + [nil, Types::STOP, 0] + ) + prot.should_receive(:read_string).ordered.and_return "test message" + prot.should_receive(:read_i32).ordered.and_return ApplicationException::BAD_SEQUENCE_ID + prot.should_receive(:read_field_end).exactly(2).times + prot.should_receive(:read_struct_end).ordered + + e = ApplicationException.new + e.read(prot) + e.message.should == "test message" + e.type.should == ApplicationException::BAD_SEQUENCE_ID + end + + it "should skip bad fields when reading a struct" do + prot = mock("MockProtocol") + prot.should_receive(:read_struct_begin).ordered + prot.should_receive(:read_field_begin).exactly(5).times.and_return( + ["type", Types::I32, 2], + ["type", Types::STRING, 2], + ["message", Types::MAP, 1], + ["message", Types::STRING, 3], + [nil, Types::STOP, 0] + ) + prot.should_receive(:read_i32).and_return ApplicationException::INVALID_MESSAGE_TYPE + prot.should_receive(:skip).with(Types::STRING).twice + prot.should_receive(:skip).with(Types::MAP) + prot.should_receive(:read_field_end).exactly(4).times + prot.should_receive(:read_struct_end).ordered + + e = ApplicationException.new + e.read(prot) + e.message.should be_nil + e.type.should == ApplicationException::INVALID_MESSAGE_TYPE + end + + it "should write a Thrift::ApplicationException struct to the oprot" do + prot = mock("MockProtocol") + prot.should_receive(:write_struct_begin).with("Thrift::ApplicationException").ordered + prot.should_receive(:write_field_begin).with("message", Types::STRING, 1).ordered + prot.should_receive(:write_string).with("test message").ordered + prot.should_receive(:write_field_begin).with("type", Types::I32, 2).ordered + prot.should_receive(:write_i32).with(ApplicationException::UNKNOWN_METHOD).ordered + prot.should_receive(:write_field_end).twice + prot.should_receive(:write_field_stop).ordered + prot.should_receive(:write_struct_end).ordered + + e = ApplicationException.new(ApplicationException::UNKNOWN_METHOD, "test message") + e.write(prot) + end + + it "should skip nil fields when writing to the oprot" do + prot = mock("MockProtocol") + prot.should_receive(:write_struct_begin).with("Thrift::ApplicationException").ordered + prot.should_receive(:write_field_begin).with("message", Types::STRING, 1).ordered + prot.should_receive(:write_string).with("test message").ordered + prot.should_receive(:write_field_end).ordered + prot.should_receive(:write_field_stop).ordered + prot.should_receive(:write_struct_end).ordered + + e = ApplicationException.new(nil, "test message") + e.write(prot) + + prot = mock("MockProtocol") + prot.should_receive(:write_struct_begin).with("Thrift::ApplicationException").ordered + prot.should_receive(:write_field_begin).with("type", Types::I32, 2).ordered + prot.should_receive(:write_i32).with(ApplicationException::BAD_SEQUENCE_ID).ordered + prot.should_receive(:write_field_end).ordered + prot.should_receive(:write_field_stop).ordered + prot.should_receive(:write_struct_end).ordered + + e = ApplicationException.new(ApplicationException::BAD_SEQUENCE_ID) + e.write(prot) + + prot = mock("MockProtocol") + prot.should_receive(:write_struct_begin).with("Thrift::ApplicationException").ordered + prot.should_receive(:write_field_stop).ordered + prot.should_receive(:write_struct_end).ordered + + e = ApplicationException.new(nil) + e.write(prot) + end + end + + describe ProtocolException do + it "should have an accessible type" do + prot = ProtocolException.new(ProtocolException::SIZE_LIMIT, "message") + prot.type.should == ProtocolException::SIZE_LIMIT + prot.message.should == "message" + end + end +end diff --git a/lib/rb/spec/http_client_spec.rb b/lib/rb/spec/http_client_spec.rb new file mode 100644 index 00000000..94526deb --- /dev/null +++ b/lib/rb/spec/http_client_spec.rb @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' + +class ThriftHTTPClientTransportSpec < Spec::ExampleGroup + include Thrift + + describe HTTPClientTransport do + before(:each) do + @client = HTTPClientTransport.new("http://my.domain.com/path/to/service") + end + + it "should always be open" do + @client.should be_open + @client.close + @client.should be_open + end + + it "should post via HTTP and return the results" do + @client.write "a test" + @client.write " frame" + Net::HTTP.should_receive(:new).with("my.domain.com", 80).and_return do + mock("Net::HTTP").tee do |http| + http.should_receive(:use_ssl=).with(false) + http.should_receive(:post).with("/path/to/service", "a test frame", {"Content-Type"=>"application/x-thrift"}).and_return([nil, "data"]) + end + end + @client.flush + @client.read(10).should == "data" + end + end +end diff --git a/lib/rb/spec/mongrel_http_server_spec.rb b/lib/rb/spec/mongrel_http_server_spec.rb new file mode 100644 index 00000000..c994491c --- /dev/null +++ b/lib/rb/spec/mongrel_http_server_spec.rb @@ -0,0 +1,117 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' +require 'thrift/server/mongrel_http_server' + +class ThriftHTTPServerSpec < Spec::ExampleGroup + include Thrift + + Handler = MongrelHTTPServer::Handler + + describe MongrelHTTPServer do + it "should have appropriate defaults" do + mock_factory = mock("BinaryProtocolFactory") + mock_proc = mock("Processor") + BinaryProtocolFactory.should_receive(:new).and_return(mock_factory) + Mongrel::HttpServer.should_receive(:new).with("0.0.0.0", 80).and_return do + mock("Mongrel::HttpServer").tee do |mock| + handler = mock("Handler") + Handler.should_receive(:new).with(mock_proc, mock_factory).and_return(handler) + mock.should_receive(:register).with("/", handler) + end + end + MongrelHTTPServer.new(mock_proc) + end + + it "should understand :ip, :port, :path, and :protocol_factory" do + mock_proc = mock("Processor") + mock_factory = mock("ProtocolFactory") + Mongrel::HttpServer.should_receive(:new).with("1.2.3.4", 1234).and_return do + mock("Mongrel::HttpServer").tee do |mock| + handler = mock("Handler") + Handler.should_receive(:new).with(mock_proc, mock_factory).and_return(handler) + mock.should_receive(:register).with("/foo", handler) + end + end + MongrelHTTPServer.new(mock_proc, :ip => "1.2.3.4", :port => 1234, :path => "foo", + :protocol_factory => mock_factory) + end + + it "should serve using Mongrel::HttpServer" do + BinaryProtocolFactory.stub!(:new) + Mongrel::HttpServer.should_receive(:new).and_return do + mock("Mongrel::HttpServer").tee do |mock| + Handler.stub!(:new) + mock.stub!(:register) + mock.should_receive(:run).and_return do + mock("Mongrel::HttpServer.run").tee do |runner| + runner.should_receive(:join) + end + end + end + end + MongrelHTTPServer.new(nil).serve + end + end + + describe MongrelHTTPServer::Handler do + before(:each) do + @processor = mock("Processor") + @factory = mock("ProtocolFactory") + @handler = Handler.new(@processor, @factory) + end + + it "should return 404 for non-POST requests" do + request = mock("request", :params => {"REQUEST_METHOD" => "GET"}) + response = mock("response") + response.should_receive(:start).with(404) + response.should_not_receive(:start).with(200) + @handler.process(request, response) + end + + it "should serve using application/x-thrift" do + request = mock("request", :params => {"REQUEST_METHOD" => "POST"}, :body => nil) + response = mock("response") + head = mock("head") + head.should_receive(:[]=).with("Content-Type", "application/x-thrift") + IOStreamTransport.stub!(:new) + @factory.stub!(:get_protocol) + @processor.stub!(:process) + response.should_receive(:start).with(200).and_yield(head, nil) + @handler.process(request, response) + end + + it "should use the IOStreamTransport" do + body = mock("body") + request = mock("request", :params => {"REQUEST_METHOD" => "POST"}, :body => body) + response = mock("response") + head = mock("head") + head.stub!(:[]=) + out = mock("out") + protocol = mock("protocol") + transport = mock("transport") + IOStreamTransport.should_receive(:new).with(body, out).and_return(transport) + @factory.should_receive(:get_protocol).with(transport).and_return(protocol) + @processor.should_receive(:process).with(protocol, protocol) + response.should_receive(:start).with(200).and_yield(head, out) + @handler.process(request, response) + end + end +end diff --git a/lib/rb/spec/nonblocking_server_spec.rb b/lib/rb/spec/nonblocking_server_spec.rb new file mode 100644 index 00000000..a0e86cf2 --- /dev/null +++ b/lib/rb/spec/nonblocking_server_spec.rb @@ -0,0 +1,266 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' +require File.dirname(__FILE__) + '/gen-rb/nonblocking_service' + +class ThriftNonblockingServerSpec < Spec::ExampleGroup + include Thrift + include SpecNamespace + + class Handler + def initialize + @queue = Queue.new + end + + attr_accessor :server + + def greeting(english) + if english + SpecNamespace::Hello.new + else + SpecNamespace::Hello.new(:greeting => "Aloha!") + end + end + + def block + @queue.pop + end + + def unblock(n) + n.times { @queue.push true } + end + + def sleep(time) + Kernel.sleep time + end + + def shutdown + @server.shutdown(0, false) + end + end + + class SpecTransport < BaseTransport + def initialize(transport, queue) + @transport = transport + @queue = queue + @flushed = false + end + + def open? + @transport.open? + end + + def open + @transport.open + end + + def close + @transport.close + end + + def read(sz) + @transport.read(sz) + end + + def write(buf,sz=nil) + @transport.write(buf, sz) + end + + def flush + @queue.push :flushed unless @flushed or @queue.nil? + @flushed = true + @transport.flush + end + end + + class SpecServerSocket < ServerSocket + def initialize(host, port, queue) + super(host, port) + @queue = queue + end + + def listen + super + @queue.push :listen + end + end + + describe Thrift::NonblockingServer do + before(:each) do + @port = 43251 + handler = Handler.new + processor = NonblockingService::Processor.new(handler) + queue = Queue.new + @transport = SpecServerSocket.new('localhost', @port, queue) + transport_factory = FramedTransportFactory.new + logger = Logger.new(STDERR) + logger.level = Logger::WARN + @server = NonblockingServer.new(processor, @transport, transport_factory, nil, 5, logger) + handler.server = @server + @server_thread = Thread.new(Thread.current) do |master_thread| + begin + @server.serve + rescue => e + p e + puts e.backtrace * "\n" + master_thread.raise e + end + end + queue.pop + + @clients = [] + @catch_exceptions = false + end + + after(:each) do + @clients.each { |client, trans| trans.close } + # @server.shutdown(1) + @server_thread.kill + @transport.close + end + + def setup_client(queue = nil) + transport = SpecTransport.new(FramedTransport.new(Socket.new('localhost', @port)), queue) + protocol = BinaryProtocol.new(transport) + client = NonblockingService::Client.new(protocol) + transport.open + @clients << [client, transport] + client + end + + def setup_client_thread(result) + queue = Queue.new + Thread.new do + begin + client = setup_client + while (cmd = queue.pop) + msg, *args = cmd + case msg + when :block + result << client.block + when :unblock + client.unblock(args.first) + when :hello + result << client.greeting(true) # ignore result + when :sleep + client.sleep(args[0] || 0.5) + result << :slept + when :shutdown + client.shutdown + when :exit + result << :done + break + end + end + @clients.each { |c,t| t.close and break if c == client } #close the transport + rescue => e + raise e unless @catch_exceptions + end + end + queue + end + + it "should handle basic message passing" do + client = setup_client + client.greeting(true).should == Hello.new + client.greeting(false).should == Hello.new(:greeting => 'Aloha!') + @server.shutdown + end + + it "should handle concurrent clients" do + queue = Queue.new + trans_queue = Queue.new + 4.times do + Thread.new(Thread.current) do |main_thread| + begin + queue.push setup_client(trans_queue).block + rescue => e + main_thread.raise e + end + end + end + 4.times { trans_queue.pop } + setup_client.unblock(4) + 4.times { queue.pop.should be_true } + @server.shutdown + end + + it "should handle messages from more than 5 long-lived connections" do + queues = [] + result = Queue.new + 7.times do |i| + queues << setup_client_thread(result) + Thread.pass if i == 4 # give the server time to accept connections + end + client = setup_client + # block 4 connections + 4.times { |i| queues[i] << :block } + queues[4] << :hello + queues[5] << :hello + queues[6] << :hello + 3.times { result.pop.should == Hello.new } + client.greeting(true).should == Hello.new + queues[5] << [:unblock, 4] + 4.times { result.pop.should be_true } + queues[2] << :hello + result.pop.should == Hello.new + client.greeting(false).should == Hello.new(:greeting => 'Aloha!') + 7.times { queues.shift << :exit } + client.greeting(true).should == Hello.new + @server.shutdown + end + + it "should shut down when asked" do + # connect first to ensure it's running + client = setup_client + client.greeting(false) # force a message pass + @server.shutdown + @server_thread.join(2).should be_an_instance_of(Thread) + end + + it "should continue processing active messages when shutting down" do + result = Queue.new + client = setup_client_thread(result) + client << :sleep + sleep 0.1 # give the server time to start processing the client's message + @server.shutdown + @server_thread.join(2).should be_an_instance_of(Thread) + result.pop.should == :slept + end + + it "should kill active messages when they don't expire while shutting down" do + result = Queue.new + client = setup_client_thread(result) + client << [:sleep, 10] + sleep 0.1 # start processing the client's message + @server.shutdown(1) + @catch_exceptions = true + @server_thread.join(3).should_not be_nil + result.should be_empty + end + + it "should allow shutting down in response to a message" do + client = setup_client + client.greeting(true).should == Hello.new + client.shutdown + @server_thread.join(2).should_not be_nil + end + end +end diff --git a/lib/rb/spec/processor_spec.rb b/lib/rb/spec/processor_spec.rb new file mode 100644 index 00000000..d35f6528 --- /dev/null +++ b/lib/rb/spec/processor_spec.rb @@ -0,0 +1,83 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' + +class ThriftProcessorSpec < Spec::ExampleGroup + include Thrift + + class ProcessorSpec + include Thrift::Processor + end + + describe "Processor" do + before(:each) do + @processor = ProcessorSpec.new(mock("MockHandler")) + @prot = mock("MockProtocol") + end + + def mock_trans(obj) + obj.should_receive(:trans).ordered.and_return do + mock("trans").tee do |trans| + trans.should_receive(:flush).ordered + end + end + end + + it "should call process_ when it receives that message" do + @prot.should_receive(:read_message_begin).ordered.and_return ['testMessage', MessageTypes::CALL, 17] + @processor.should_receive(:process_testMessage).with(17, @prot, @prot).ordered + @processor.process(@prot, @prot).should == true + end + + it "should raise an ApplicationException when the received message cannot be processed" do + @prot.should_receive(:read_message_begin).ordered.and_return ['testMessage', MessageTypes::CALL, 4] + @prot.should_receive(:skip).with(Types::STRUCT).ordered + @prot.should_receive(:read_message_end).ordered + @prot.should_receive(:write_message_begin).with('testMessage', MessageTypes::EXCEPTION, 4).ordered + ApplicationException.should_receive(:new).with(ApplicationException::UNKNOWN_METHOD, "Unknown function testMessage").and_return do + mock(ApplicationException).tee do |e| + e.should_receive(:write).with(@prot).ordered + end + end + @prot.should_receive(:write_message_end).ordered + mock_trans(@prot) + @processor.process(@prot, @prot) + end + + it "should pass args off to the args class" do + args_class = mock("MockArgsClass") + args = mock("#").tee do |args| + args.should_receive(:read).with(@prot).ordered + end + args_class.should_receive(:new).and_return args + @prot.should_receive(:read_message_end).ordered + @processor.read_args(@prot, args_class).should eql(args) + end + + it "should write out a reply when asked" do + @prot.should_receive(:write_message_begin).with('testMessage', MessageTypes::REPLY, 23).ordered + result = mock("MockResult") + result.should_receive(:write).with(@prot).ordered + @prot.should_receive(:write_message_end).ordered + mock_trans(@prot) + @processor.write_result(result, @prot, 'testMessage', 23) + end + end +end diff --git a/lib/rb/spec/serializer_spec.rb b/lib/rb/spec/serializer_spec.rb new file mode 100644 index 00000000..82f374b1 --- /dev/null +++ b/lib/rb/spec/serializer_spec.rb @@ -0,0 +1,70 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' +require File.dirname(__FILE__) + '/gen-rb/thrift_spec_types' + +class ThriftSerializerSpec < Spec::ExampleGroup + include Thrift + include SpecNamespace + + describe Serializer do + it "should serialize structs to binary by default" do + serializer = Serializer.new(Thrift::BinaryProtocolAcceleratedFactory.new) + data = serializer.serialize(Hello.new(:greeting => "'Ello guv'nor!")) + data.should == "\x0B\x00\x01\x00\x00\x00\x0E'Ello guv'nor!\x00" + end + + it "should serialize structs to the given protocol" do + protocol = BaseProtocol.new(mock("transport")) + protocol.should_receive(:write_struct_begin).with("SpecNamespace::Hello") + protocol.should_receive(:write_field_begin).with("greeting", Types::STRING, 1) + protocol.should_receive(:write_string).with("Good day") + protocol.should_receive(:write_field_end) + protocol.should_receive(:write_field_stop) + protocol.should_receive(:write_struct_end) + protocol_factory = mock("ProtocolFactory") + protocol_factory.stub!(:get_protocol).and_return(protocol) + serializer = Serializer.new(protocol_factory) + serializer.serialize(Hello.new(:greeting => "Good day")) + end + end + + describe Deserializer do + it "should deserialize structs from binary by default" do + deserializer = Deserializer.new + data = "\x0B\x00\x01\x00\x00\x00\x0E'Ello guv'nor!\x00" + deserializer.deserialize(Hello.new, data).should == Hello.new(:greeting => "'Ello guv'nor!") + end + + it "should deserialize structs from the given protocol" do + protocol = BaseProtocol.new(mock("transport")) + protocol.should_receive(:read_struct_begin).and_return("SpecNamespace::Hello") + protocol.should_receive(:read_field_begin).and_return(["greeting", Types::STRING, 1], + [nil, Types::STOP, 0]) + protocol.should_receive(:read_string).and_return("Good day") + protocol.should_receive(:read_field_end) + protocol.should_receive(:read_struct_end) + protocol_factory = mock("ProtocolFactory") + protocol_factory.stub!(:get_protocol).and_return(protocol) + deserializer = Deserializer.new(protocol_factory) + deserializer.deserialize(Hello.new, "").should == Hello.new(:greeting => "Good day") + end + end +end diff --git a/lib/rb/spec/server_socket_spec.rb b/lib/rb/spec/server_socket_spec.rb new file mode 100644 index 00000000..fce50134 --- /dev/null +++ b/lib/rb/spec/server_socket_spec.rb @@ -0,0 +1,80 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' +require File.dirname(__FILE__) + "/socket_spec_shared" + +class ThriftServerSocketSpec < Spec::ExampleGroup + include Thrift + + describe ServerSocket do + before(:each) do + @socket = ServerSocket.new(1234) + end + + it "should create a handle when calling listen" do + TCPServer.should_receive(:new).with(nil, 1234) + @socket.listen + end + + it "should accept an optional host argument" do + @socket = ServerSocket.new('localhost', 1234) + TCPServer.should_receive(:new).with('localhost', 1234) + @socket.listen + end + + it "should create a Thrift::Socket to wrap accepted sockets" do + handle = mock("TCPServer") + TCPServer.should_receive(:new).with(nil, 1234).and_return(handle) + @socket.listen + sock = mock("sock") + handle.should_receive(:accept).and_return(sock) + trans = mock("Socket") + Socket.should_receive(:new).and_return(trans) + trans.should_receive(:handle=).with(sock) + @socket.accept.should == trans + end + + it "should close the handle when closed" do + handle = mock("TCPServer", :closed? => false) + TCPServer.should_receive(:new).with(nil, 1234).and_return(handle) + @socket.listen + handle.should_receive(:close) + @socket.close + end + + it "should return nil when accepting if there is no handle" do + @socket.accept.should be_nil + end + + it "should return true for closed? when appropriate" do + handle = mock("TCPServer", :closed? => false) + TCPServer.stub!(:new).and_return(handle) + @socket.listen + @socket.should_not be_closed + handle.stub!(:close) + @socket.close + @socket.should be_closed + @socket.listen + @socket.should_not be_closed + handle.stub!(:closed?).and_return(true) + @socket.should be_closed + end + end +end diff --git a/lib/rb/spec/server_spec.rb b/lib/rb/spec/server_spec.rb new file mode 100644 index 00000000..ffe9bffa --- /dev/null +++ b/lib/rb/spec/server_spec.rb @@ -0,0 +1,160 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' + +class ThriftServerSpec < Spec::ExampleGroup + include Thrift + + describe BaseServer do + it "should default to BaseTransportFactory and BinaryProtocolFactory when not specified" do + server = BaseServer.new(mock("Processor"), mock("BaseServerTransport")) + server.instance_variable_get(:'@transport_factory').should be_an_instance_of(BaseTransportFactory) + server.instance_variable_get(:'@protocol_factory').should be_an_instance_of(BinaryProtocolFactory) + end + + # serve is a noop, so can't test that + end + + shared_examples_for "servers" do + before(:each) do + @processor = mock("Processor") + @serverTrans = mock("ServerTransport") + @trans = mock("BaseTransport") + @prot = mock("BaseProtocol") + @client = mock("Client") + @server = server_type.new(@processor, @serverTrans, @trans, @prot) + end + end + + describe SimpleServer do + it_should_behave_like "servers" + + def server_type + SimpleServer + end + + it "should serve in the main thread" do + @serverTrans.should_receive(:listen).ordered + @serverTrans.should_receive(:accept).exactly(3).times.and_return(@client) + @trans.should_receive(:get_transport).exactly(3).times.with(@client).and_return(@trans) + @prot.should_receive(:get_protocol).exactly(3).times.with(@trans).and_return(@prot) + x = 0 + @processor.should_receive(:process).exactly(3).times.with(@prot, @prot).and_return do + case (x += 1) + when 1 then raise Thrift::TransportException + when 2 then raise Thrift::ProtocolException + when 3 then throw :stop + end + end + @trans.should_receive(:close).exactly(3).times + @serverTrans.should_receive(:close).ordered + lambda { @server.serve }.should throw_symbol(:stop) + end + end + + describe ThreadedServer do + it_should_behave_like "servers" + + def server_type + ThreadedServer + end + + it "should serve using threads" do + @serverTrans.should_receive(:listen).ordered + @serverTrans.should_receive(:accept).exactly(3).times.and_return(@client) + @trans.should_receive(:get_transport).exactly(3).times.with(@client).and_return(@trans) + @prot.should_receive(:get_protocol).exactly(3).times.with(@trans).and_return(@prot) + Thread.should_receive(:new).with(@prot, @trans).exactly(3).times.and_yield(@prot, @trans) + x = 0 + @processor.should_receive(:process).exactly(3).times.with(@prot, @prot).and_return do + case (x += 1) + when 1 then raise Thrift::TransportException + when 2 then raise Thrift::ProtocolException + when 3 then throw :stop + end + end + @trans.should_receive(:close).exactly(3).times + @serverTrans.should_receive(:close).ordered + lambda { @server.serve }.should throw_symbol(:stop) + end + end + + describe ThreadPoolServer do + it_should_behave_like "servers" + + def server_type + # put this stuff here so it runs before the server is created + @threadQ = mock("SizedQueue") + SizedQueue.should_receive(:new).with(20).and_return(@threadQ) + @excQ = mock("Queue") + Queue.should_receive(:new).and_return(@excQ) + ThreadPoolServer + end + + it "should set up the queues" do + @server.instance_variable_get(:'@thread_q').should be(@threadQ) + @server.instance_variable_get(:'@exception_q').should be(@excQ) + end + + it "should serve inside a thread" do + Thread.should_receive(:new).and_return do |block| + @server.should_receive(:serve) + block.call + @server.rspec_verify + end + @excQ.should_receive(:pop).and_throw(:popped) + lambda { @server.rescuable_serve }.should throw_symbol(:popped) + end + + it "should avoid running the server twice when retrying rescuable_serve" do + Thread.should_receive(:new).and_return do |block| + @server.should_receive(:serve) + block.call + @server.rspec_verify + end + @excQ.should_receive(:pop).twice.and_throw(:popped) + lambda { @server.rescuable_serve }.should throw_symbol(:popped) + lambda { @server.rescuable_serve }.should throw_symbol(:popped) + end + + it "should serve using a thread pool" do + @serverTrans.should_receive(:listen).ordered + @threadQ.should_receive(:push).with(:token) + @threadQ.should_receive(:pop) + Thread.should_receive(:new).and_yield + @serverTrans.should_receive(:accept).exactly(3).times.and_return(@client) + @trans.should_receive(:get_transport).exactly(3).times.and_return(@trans) + @prot.should_receive(:get_protocol).exactly(3).times.and_return(@prot) + x = 0 + error = RuntimeError.new("Stopped") + @processor.should_receive(:process).exactly(3).times.with(@prot, @prot).and_return do + case (x += 1) + when 1 then raise Thrift::TransportException + when 2 then raise Thrift::ProtocolException + when 3 then raise error + end + end + @trans.should_receive(:close).exactly(3).times + @excQ.should_receive(:push).with(error).and_throw(:stop) + @serverTrans.should_receive(:close) + lambda { @server.serve }.should throw_symbol(:stop) + end + end +end diff --git a/lib/rb/spec/socket_spec.rb b/lib/rb/spec/socket_spec.rb new file mode 100644 index 00000000..dd8b0f92 --- /dev/null +++ b/lib/rb/spec/socket_spec.rb @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' +require File.dirname(__FILE__) + "/socket_spec_shared" + +class ThriftSocketSpec < Spec::ExampleGroup + include Thrift + + describe Socket do + before(:each) do + @socket = Socket.new + @handle = mock("Handle", :closed? => false) + @handle.stub!(:close) + @handle.stub!(:connect_nonblock) + ::Socket.stub!(:new).and_return(@handle) + end + + it_should_behave_like "a socket" + + it "should raise a TransportException when it cannot open a socket" do + ::Socket.should_receive(:new).and_raise(StandardError) + lambda { @socket.open }.should raise_error(Thrift::TransportException) { |e| e.type.should == Thrift::TransportException::NOT_OPEN } + end + + it "should open a ::Socket with default args" do + ::Socket.should_receive(:new).and_return(mock("Handle", :connect_nonblock => true)) + ::Socket.should_receive(:getaddrinfo).with("localhost", 9090).and_return([[]]) + ::Socket.should_receive(:sockaddr_in) + @socket.open + end + + it "should accept host/port options" do + ::Socket.should_receive(:new).and_return(mock("Handle", :connect_nonblock => true)) + ::Socket.should_receive(:getaddrinfo).with("my.domain", 1234).and_return([[]]) + ::Socket.should_receive(:sockaddr_in) + Socket.new('my.domain', 1234).open + end + + it "should accept an optional timeout" do + ::Socket.stub!(:new) + Socket.new('localhost', 8080, 5).timeout.should == 5 + end + end +end diff --git a/lib/rb/spec/socket_spec_shared.rb b/lib/rb/spec/socket_spec_shared.rb new file mode 100644 index 00000000..96b433b8 --- /dev/null +++ b/lib/rb/spec/socket_spec_shared.rb @@ -0,0 +1,104 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' + +shared_examples_for "a socket" do + it "should open a socket" do + @socket.open.should == @handle + end + + it "should be open whenever it has a handle" do + @socket.should_not be_open + @socket.open + @socket.should be_open + @socket.handle = nil + @socket.should_not be_open + @socket.handle = @handle + @socket.close + @socket.should_not be_open + end + + it "should write data to the handle" do + @socket.open + @handle.should_receive(:write).with("foobar") + @socket.write("foobar") + @handle.should_receive(:write).with("fail").and_raise(StandardError) + lambda { @socket.write("fail") }.should raise_error(Thrift::TransportException) { |e| e.type.should == Thrift::TransportException::NOT_OPEN } + end + + it "should raise an error when it cannot read from the handle" do + @socket.open + @handle.should_receive(:readpartial).with(17).and_raise(StandardError) + lambda { @socket.read(17) }.should raise_error(Thrift::TransportException) { |e| e.type.should == Thrift::TransportException::NOT_OPEN } + end + + it "should return the data read when reading from the handle works" do + @socket.open + @handle.should_receive(:readpartial).with(17).and_return("test data") + @socket.read(17).should == "test data" + end + + it "should declare itself as closed when it has an error" do + @socket.open + @handle.should_receive(:write).with("fail").and_raise(StandardError) + @socket.should be_open + lambda { @socket.write("fail") }.should raise_error + @socket.should_not be_open + end + + it "should raise an error when the stream is closed" do + @socket.open + @handle.stub!(:closed?).and_return(true) + @socket.should_not be_open + lambda { @socket.write("fail") }.should raise_error(IOError, "closed stream") + lambda { @socket.read(10) }.should raise_error(IOError, "closed stream") + end + + it "should support the timeout accessor for read" do + @socket.timeout = 3 + @socket.open + IO.should_receive(:select).with([@handle], nil, nil, 3).and_return([[@handle], [], []]) + @handle.should_receive(:readpartial).with(17).and_return("test data") + @socket.read(17).should == "test data" + end + + it "should support the timeout accessor for write" do + @socket.timeout = 3 + @socket.open + IO.should_receive(:select).with(nil, [@handle], nil, 3).twice.and_return([[], [@handle], []]) + @handle.should_receive(:write_nonblock).with("test data").and_return(4) + @handle.should_receive(:write_nonblock).with(" data").and_return(5) + @socket.write("test data").should == 9 + end + + it "should raise an error when read times out" do + @socket.timeout = 0.5 + @socket.open + IO.should_receive(:select).with([@handle], nil, nil, 0.5).at_least(1).times.and_return(nil) + lambda { @socket.read(17) }.should raise_error(Thrift::TransportException) { |e| e.type.should == Thrift::TransportException::TIMED_OUT } + end + + it "should raise an error when write times out" do + @socket.timeout = 0.5 + @socket.open + IO.should_receive(:select).with(nil, [@handle], nil, 0.5).any_number_of_times.and_return(nil) + lambda { @socket.write("test data") }.should raise_error(Thrift::TransportException) { |e| e.type.should == Thrift::TransportException::TIMED_OUT } + end +end diff --git a/lib/rb/spec/spec_helper.rb b/lib/rb/spec/spec_helper.rb new file mode 100644 index 00000000..3c20bf99 --- /dev/null +++ b/lib/rb/spec/spec_helper.rb @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'rubygems' +# require at least 1.1.4 to fix a bug with describing Modules +gem 'rspec', '>= 1.1.4' +require 'spec' + +$:.unshift File.join(File.dirname(__FILE__), *%w[.. ext]) + +# pretend we already loaded fastthread, otherwise the nonblocking_server_spec +# will get screwed up +# $" << 'fastthread.bundle' + +require File.dirname(__FILE__) + '/../lib/thrift' + +class Object + # tee is a useful method, so let's let our tests have it + def tee(&block) + block.call(self) + self + end +end + +Spec::Runner.configure do |configuration| + configuration.before(:each) do + Thrift.type_checking = true + end +end + +require File.dirname(__FILE__) + "/../debug_proto_test/gen-rb/Srv" +require File.dirname(__FILE__) + "/../debug_proto_test/gen-rb/debug_proto_test_constants" + +module Fixtures + COMPACT_PROTOCOL_TEST_STRUCT = COMPACT_TEST.dup + COMPACT_PROTOCOL_TEST_STRUCT.a_binary = [0,1,2,3,4,5,6,7,8].pack('c*') + COMPACT_PROTOCOL_TEST_STRUCT.set_byte_map = nil + COMPACT_PROTOCOL_TEST_STRUCT.map_byte_map = nil +end \ No newline at end of file diff --git a/lib/rb/spec/struct_spec.rb b/lib/rb/spec/struct_spec.rb new file mode 100644 index 00000000..23a701ec --- /dev/null +++ b/lib/rb/spec/struct_spec.rb @@ -0,0 +1,253 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' +require File.dirname(__FILE__) + '/gen-rb/thrift_spec_types' + +class ThriftStructSpec < Spec::ExampleGroup + include Thrift + include SpecNamespace + + describe Struct do + it "should iterate over all fields properly" do + fields = {} + Foo.new.each_field { |fid,field_info| fields[fid] = field_info } + fields.should == Foo::FIELDS + end + + it "should initialize all fields to defaults" do + struct = Foo.new + struct.simple.should == 53 + struct.words.should == "words" + struct.hello.should == Hello.new(:greeting => 'hello, world!') + struct.ints.should == [1, 2, 2, 3] + struct.complex.should be_nil + struct.shorts.should == Set.new([5, 17, 239]) + end + + it "should not share default values between instances" do + begin + struct = Foo.new + struct.ints << 17 + Foo.new.ints.should == [1,2,2,3] + ensure + # ensure no leakage to other tests + Foo::FIELDS[4][:default] = [1,2,2,3] + end + end + + it "should properly initialize boolean values" do + struct = BoolStruct.new(:yesno => false) + struct.yesno.should be_false + end + + it "should have proper == semantics" do + Foo.new.should_not == Hello.new + Foo.new.should == Foo.new + Foo.new(:simple => 52).should_not == Foo.new + end + + it "should read itself off the wire" do + struct = Foo.new + prot = BaseProtocol.new(mock("transport")) + prot.should_receive(:read_struct_begin).twice + prot.should_receive(:read_struct_end).twice + prot.should_receive(:read_field_begin).and_return( + ['complex', Types::MAP, 5], # Foo + ['words', Types::STRING, 2], # Foo + ['hello', Types::STRUCT, 3], # Foo + ['greeting', Types::STRING, 1], # Hello + [nil, Types::STOP, 0], # Hello + ['simple', Types::I32, 1], # Foo + ['ints', Types::LIST, 4], # Foo + ['shorts', Types::SET, 6], # Foo + [nil, Types::STOP, 0] # Hello + ) + prot.should_receive(:read_field_end).exactly(7).times + prot.should_receive(:read_map_begin).and_return( + [Types::I32, Types::MAP, 2], # complex + [Types::STRING, Types::DOUBLE, 2], # complex/1/value + [Types::STRING, Types::DOUBLE, 1] # complex/2/value + ) + prot.should_receive(:read_map_end).exactly(3).times + prot.should_receive(:read_list_begin).and_return([Types::I32, 4]) + prot.should_receive(:read_list_end) + prot.should_receive(:read_set_begin).and_return([Types::I16, 2]) + prot.should_receive(:read_set_end) + prot.should_receive(:read_i32).and_return( + 1, 14, # complex keys + 42, # simple + 4, 23, 4, 29 # ints + ) + prot.should_receive(:read_string).and_return("pi", "e", "feigenbaum", "apple banana", "what's up?") + prot.should_receive(:read_double).and_return(Math::PI, Math::E, 4.669201609) + prot.should_receive(:read_i16).and_return(2, 3) + prot.should_not_receive(:skip) + struct.read(prot) + + struct.simple.should == 42 + struct.complex.should == {1 => {"pi" => Math::PI, "e" => Math::E}, 14 => {"feigenbaum" => 4.669201609}} + struct.hello.should == Hello.new(:greeting => "what's up?") + struct.words.should == "apple banana" + struct.ints.should == [4, 23, 4, 29] + struct.shorts.should == Set.new([3, 2]) + end + + it "should skip unexpected fields in structs and use default values" do + struct = Foo.new + prot = BaseProtocol.new(mock("transport")) + prot.should_receive(:read_struct_begin) + prot.should_receive(:read_struct_end) + prot.should_receive(:read_field_begin).and_return( + ['simple', Types::I32, 1], + ['complex', Types::STRUCT, 5], + ['thinz', Types::MAP, 7], + ['foobar', Types::I32, 3], + ['words', Types::STRING, 2], + [nil, Types::STOP, 0] + ) + prot.should_receive(:read_field_end).exactly(5).times + prot.should_receive(:read_i32).and_return(42) + prot.should_receive(:read_string).and_return("foobar") + prot.should_receive(:skip).with(Types::STRUCT) + prot.should_receive(:skip).with(Types::MAP) + # prot.should_receive(:read_map_begin).and_return([Types::I32, Types::I32, 0]) + # prot.should_receive(:read_map_end) + prot.should_receive(:skip).with(Types::I32) + struct.read(prot) + + struct.simple.should == 42 + struct.complex.should be_nil + struct.words.should == "foobar" + struct.hello.should == Hello.new(:greeting => 'hello, world!') + struct.ints.should == [1, 2, 2, 3] + struct.shorts.should == Set.new([5, 17, 239]) + end + + it "should write itself to the wire" do + prot = BaseProtocol.new(mock("transport")) #mock("Protocol") + prot.should_receive(:write_struct_begin).with("SpecNamespace::Foo") + prot.should_receive(:write_struct_begin).with("SpecNamespace::Hello") + prot.should_receive(:write_struct_end).twice + prot.should_receive(:write_field_begin).with('ints', Types::LIST, 4) + prot.should_receive(:write_i32).with(1) + prot.should_receive(:write_i32).with(2).twice + prot.should_receive(:write_i32).with(3) + prot.should_receive(:write_field_begin).with('complex', Types::MAP, 5) + prot.should_receive(:write_i32).with(5) + prot.should_receive(:write_string).with('foo') + prot.should_receive(:write_double).with(1.23) + prot.should_receive(:write_field_begin).with('shorts', Types::SET, 6) + prot.should_receive(:write_i16).with(5) + prot.should_receive(:write_i16).with(17) + prot.should_receive(:write_i16).with(239) + prot.should_receive(:write_field_stop).twice + prot.should_receive(:write_field_end).exactly(6).times + prot.should_receive(:write_field_begin).with('simple', Types::I32, 1) + prot.should_receive(:write_i32).with(53) + prot.should_receive(:write_field_begin).with('hello', Types::STRUCT, 3) + prot.should_receive(:write_field_begin).with('greeting', Types::STRING, 1) + prot.should_receive(:write_string).with('hello, world!') + prot.should_receive(:write_map_begin).with(Types::I32, Types::MAP, 1) + prot.should_receive(:write_map_begin).with(Types::STRING, Types::DOUBLE, 1) + prot.should_receive(:write_map_end).twice + prot.should_receive(:write_list_begin).with(Types::I32, 4) + prot.should_receive(:write_list_end) + prot.should_receive(:write_set_begin).with(Types::I16, 3) + prot.should_receive(:write_set_end) + + struct = Foo.new + struct.words = nil + struct.complex = {5 => {"foo" => 1.23}} + struct.write(prot) + end + + it "should raise an exception if presented with an unknown container" do + # yeah this is silly, but I'm going for code coverage here + struct = Foo.new + lambda { struct.send :write_container, nil, nil, {:type => "foo"} }.should raise_error(StandardError, "Not a container type: foo") + end + + it "should support optional type-checking in Thrift::Struct.new" do + Thrift.type_checking = true + begin + lambda { Hello.new(:greeting => 3) }.should raise_error(TypeError, "Expected Types::STRING, received Fixnum for field greeting") + ensure + Thrift.type_checking = false + end + lambda { Hello.new(:greeting => 3) }.should_not raise_error(TypeError) + end + + it "should support optional type-checking in field accessors" do + Thrift.type_checking = true + begin + hello = Hello.new + lambda { hello.greeting = 3 }.should raise_error(TypeError, "Expected Types::STRING, received Fixnum for field greeting") + ensure + Thrift.type_checking = false + end + lambda { hello.greeting = 3 }.should_not raise_error(TypeError) + end + + it "should raise an exception when unknown types are given to Thrift::Struct.new" do + lambda { Hello.new(:fish => 'salmon') }.should raise_error(Exception, "Unknown key given to SpecNamespace::Hello.new: fish") + end + + it "should support `raise Xception, 'message'` for Exception structs" do + begin + raise Xception, "something happened" + rescue Thrift::Exception => e + e.message.should == "something happened" + e.code.should == 1 + # ensure it gets serialized properly, this is the really important part + prot = BaseProtocol.new(mock("trans")) + prot.should_receive(:write_struct_begin).with("SpecNamespace::Xception") + prot.should_receive(:write_struct_end) + prot.should_receive(:write_field_begin).with('message', Types::STRING, 1)#, "something happened") + prot.should_receive(:write_string).with("something happened") + prot.should_receive(:write_field_begin).with('code', Types::I32, 2)#, 1) + prot.should_receive(:write_i32).with(1) + prot.should_receive(:write_field_stop) + prot.should_receive(:write_field_end).twice + + e.write(prot) + end + end + + it "should support the regular initializer for exception structs" do + begin + raise Xception, :message => "something happened", :code => 5 + rescue Thrift::Exception => e + e.message.should == "something happened" + e.code.should == 5 + prot = BaseProtocol.new(mock("trans")) + prot.should_receive(:write_struct_begin).with("SpecNamespace::Xception") + prot.should_receive(:write_struct_end) + prot.should_receive(:write_field_begin).with('message', Types::STRING, 1) + prot.should_receive(:write_string).with("something happened") + prot.should_receive(:write_field_begin).with('code', Types::I32, 2) + prot.should_receive(:write_i32).with(5) + prot.should_receive(:write_field_stop) + prot.should_receive(:write_field_end).twice + + e.write(prot) + end + end + end +end diff --git a/lib/rb/spec/types_spec.rb b/lib/rb/spec/types_spec.rb new file mode 100644 index 00000000..d979cfb1 --- /dev/null +++ b/lib/rb/spec/types_spec.rb @@ -0,0 +1,117 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' +require File.dirname(__FILE__) + '/gen-rb/thrift_spec_types' + +class ThriftTypesSpec < Spec::ExampleGroup + include Thrift + + before(:each) do + Thrift.type_checking = true + end + + after(:each) do + Thrift.type_checking = false + end + + describe "Type checking" do + it "should return the proper name for each type" do + Thrift.type_name(Types::I16).should == "Types::I16" + Thrift.type_name(Types::VOID).should == "Types::VOID" + Thrift.type_name(Types::LIST).should == "Types::LIST" + Thrift.type_name(42).should be_nil + end + + it "should check types properly" do + # lambda { Thrift.check_type(nil, Types::STOP) }.should raise_error(TypeError) + lambda { Thrift.check_type(3, {:type => Types::STOP}, :foo) }.should raise_error(TypeError) + lambda { Thrift.check_type(nil, {:type => Types::VOID}, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type(3, {:type => Types::VOID}, :foo) }.should raise_error(TypeError) + lambda { Thrift.check_type(true, {:type => Types::BOOL}, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type(3, {:type => Types::BOOL}, :foo) }.should raise_error(TypeError) + lambda { Thrift.check_type(42, {:type => Types::BYTE}, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type(42, {:type => Types::I16}, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type(42, {:type => Types::I32}, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type(42, {:type => Types::I64}, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type(3.14, {:type => Types::I32}, :foo) }.should raise_error(TypeError) + lambda { Thrift.check_type(3.14, {:type => Types::DOUBLE}, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type(3, {:type => Types::DOUBLE}, :foo) }.should raise_error(TypeError) + lambda { Thrift.check_type("3", {:type => Types::STRING}, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type(3, {:type => Types::STRING}, :foo) }.should raise_error(TypeError) + hello = SpecNamespace::Hello.new + lambda { Thrift.check_type(hello, {:type => Types::STRUCT, :class => SpecNamespace::Hello}, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type("foo", {:type => Types::STRUCT}, :foo) }.should raise_error(TypeError) + lambda { Thrift.check_type({:foo => 1}, {:type => Types::MAP}, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type([1], {:type => Types::MAP}, :foo) }.should raise_error(TypeError) + lambda { Thrift.check_type([1], {:type => Types::LIST}, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type({:foo => 1}, {:type => Types::LIST}, :foo) }.should raise_error(TypeError) + lambda { Thrift.check_type(Set.new([1,2]), {:type => Types::SET}, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type([1,2], {:type => Types::SET}, :foo) }.should raise_error(TypeError) + lambda { Thrift.check_type({:foo => true}, {:type => Types::SET}, :foo) }.should raise_error(TypeError) + end + + it "should error out if nil is passed and skip_types is false" do + lambda { Thrift.check_type(nil, {:type => Types::BOOL}, :foo, false) }.should raise_error(TypeError) + lambda { Thrift.check_type(nil, {:type => Types::BYTE}, :foo, false) }.should raise_error(TypeError) + lambda { Thrift.check_type(nil, {:type => Types::I16}, :foo, false) }.should raise_error(TypeError) + lambda { Thrift.check_type(nil, {:type => Types::I32}, :foo, false) }.should raise_error(TypeError) + lambda { Thrift.check_type(nil, {:type => Types::I64}, :foo, false) }.should raise_error(TypeError) + lambda { Thrift.check_type(nil, {:type => Types::DOUBLE}, :foo, false) }.should raise_error(TypeError) + lambda { Thrift.check_type(nil, {:type => Types::STRING}, :foo, false) }.should raise_error(TypeError) + lambda { Thrift.check_type(nil, {:type => Types::STRUCT}, :foo, false) }.should raise_error(TypeError) + lambda { Thrift.check_type(nil, {:type => Types::LIST}, :foo, false) }.should raise_error(TypeError) + lambda { Thrift.check_type(nil, {:type => Types::SET}, :foo, false) }.should raise_error(TypeError) + lambda { Thrift.check_type(nil, {:type => Types::MAP}, :foo, false) }.should raise_error(TypeError) + end + + it "should check element types on containers" do + field = {:type => Types::LIST, :element => {:type => Types::I32}} + lambda { Thrift.check_type([1, 2], field, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type([1, nil, 2], field, :foo) }.should raise_error(TypeError) + field = {:type => Types::MAP, :key => {:type => Types::I32}, :value => {:type => Types::STRING}} + lambda { Thrift.check_type({1 => "one", 2 => "two"}, field, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type({1 => "one", nil => "nil"}, field, :foo) }.should raise_error(TypeError) + lambda { Thrift.check_type({1 => nil, 2 => "two"}, field, :foo) }.should raise_error(TypeError) + field = {:type => Types::SET, :element => {:type => Types::I32}} + lambda { Thrift.check_type(Set.new([1, 2]), field, :foo) }.should_not raise_error(TypeError) + lambda { Thrift.check_type(Set.new([1, nil, 2]), field, :foo) }.should raise_error(TypeError) + lambda { Thrift.check_type(Set.new([1, 2.3, 2]), field, :foo) }.should raise_error(TypeError) + + field = {:type => Types::STRUCT, :class => SpecNamespace::Hello} + lambda { Thrift.check_type(SpecNamespace::BoolStruct, field, :foo) }.should raise_error(TypeError) + end + + it "should give the TypeError a readable message" do + msg = "Expected Types::STRING, received Fixnum for field foo" + lambda { Thrift.check_type(3, {:type => Types::STRING}, :foo) }.should raise_error(TypeError, msg) + msg = "Expected Types::STRING, received Fixnum for field foo.element" + field = {:type => Types::LIST, :element => {:type => Types::STRING}} + lambda { Thrift.check_type([3], field, :foo) }.should raise_error(TypeError, msg) + msg = "Expected Types::I32, received NilClass for field foo.element.key" + field = {:type => Types::LIST, + :element => {:type => Types::MAP, + :key => {:type => Types::I32}, + :value => {:type => Types::I32}}} + lambda { Thrift.check_type([{nil => 3}], field, :foo) }.should raise_error(TypeError, msg) + msg = "Expected Types::I32, received NilClass for field foo.element.value" + lambda { Thrift.check_type([{1 => nil}], field, :foo) }.should raise_error(TypeError, msg) + end + end +end diff --git a/lib/rb/spec/unix_socket_spec.rb b/lib/rb/spec/unix_socket_spec.rb new file mode 100644 index 00000000..df239d7a --- /dev/null +++ b/lib/rb/spec/unix_socket_spec.rb @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.dirname(__FILE__) + '/spec_helper' +require File.dirname(__FILE__) + "/socket_spec_shared" + +class ThriftUNIXSocketSpec < Spec::ExampleGroup + include Thrift + + describe UNIXSocket do + before(:each) do + @path = '/tmp/thrift_spec_socket' + @socket = UNIXSocket.new(@path) + @handle = mock("Handle", :closed? => false) + @handle.stub!(:close) + ::UNIXSocket.stub!(:new).and_return(@handle) + end + + it_should_behave_like "a socket" + + it "should raise a TransportException when it cannot open a socket" do + ::UNIXSocket.should_receive(:new).and_raise(StandardError) + lambda { @socket.open }.should raise_error(Thrift::TransportException) { |e| e.type.should == Thrift::TransportException::NOT_OPEN } + end + + it "should accept an optional timeout" do + ::UNIXSocket.stub!(:new) + UNIXSocket.new(@path, 5).timeout.should == 5 + end + end + + describe UNIXServerSocket do + before(:each) do + @path = '/tmp/thrift_spec_socket' + @socket = UNIXServerSocket.new(@path) + end + + it "should create a handle when calling listen" do + UNIXServer.should_receive(:new).with(@path) + @socket.listen + end + + it "should create a Thrift::UNIXSocket to wrap accepted sockets" do + handle = mock("UNIXServer") + UNIXServer.should_receive(:new).with(@path).and_return(handle) + @socket.listen + sock = mock("sock") + handle.should_receive(:accept).and_return(sock) + trans = mock("UNIXSocket") + UNIXSocket.should_receive(:new).and_return(trans) + trans.should_receive(:handle=).with(sock) + @socket.accept.should == trans + end + + it "should close the handle when closed" do + handle = mock("UNIXServer", :closed? => false) + UNIXServer.should_receive(:new).with(@path).and_return(handle) + @socket.listen + handle.should_receive(:close) + File.stub!(:delete) + @socket.close + end + + it "should delete the socket when closed" do + handle = mock("UNIXServer", :closed? => false) + UNIXServer.should_receive(:new).with(@path).and_return(handle) + @socket.listen + handle.stub!(:close) + File.should_receive(:delete).with(@path) + @socket.close + end + + it "should return nil when accepting if there is no handle" do + @socket.accept.should be_nil + end + + it "should return true for closed? when appropriate" do + handle = mock("UNIXServer", :closed? => false) + UNIXServer.stub!(:new).and_return(handle) + File.stub!(:delete) + @socket.listen + @socket.should_not be_closed + handle.stub!(:close) + @socket.close + @socket.should be_closed + @socket.listen + @socket.should_not be_closed + handle.stub!(:closed?).and_return(true) + @socket.should be_closed + end + end +end diff --git a/lib/st/README b/lib/st/README new file mode 100644 index 00000000..be865b8f --- /dev/null +++ b/lib/st/README @@ -0,0 +1,35 @@ +Thrift SmallTalk Software Library + +Last updated Nov 2007 + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Library +======= + +To get started, just file in thrift.st with Squeak, run thrift -st +on the tutorial .thrift files (and file in the resulting code), and +then: + +calc := CalculatorClient binaryOnHost: 'localhost' port: '9090' +calc addNum1: 10 num2: 15 + +Tested in Squeak 3.7, but should work fine with anything later. diff --git a/lib/st/thrift.st b/lib/st/thrift.st new file mode 100644 index 00000000..6883539f --- /dev/null +++ b/lib/st/thrift.st @@ -0,0 +1,812 @@ +" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +License); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +" + +SystemOrganization addCategory: #Thrift! +SystemOrganization addCategory: #'Thrift-Protocol'! +SystemOrganization addCategory: #'Thrift-Transport'! + +Error subclass: #TError + instanceVariableNames: 'code' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift'! + +!TError class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:28'! +signalWithCode: anInteger + self new code: anInteger; signal! ! + +!TError methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:28'! +code + ^ code! ! + +!TError methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:28'! +code: anInteger + code := anInteger! ! + +TError subclass: #TProtocolError + instanceVariableNames: '' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Protocol'! + +!TProtocolError class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 18:39'! +badVersion + ^ 4! ! + +!TProtocolError class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 18:39'! +invalidData + ^ 1! ! + +!TProtocolError class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 18:39'! +negativeSize + ^ 2! ! + +!TProtocolError class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 18:40'! +sizeLimit + ^ 3! ! + +!TProtocolError class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 18:40'! +unknown + ^ 0! ! + +TError subclass: #TTransportError + instanceVariableNames: '' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Transport'! + +TTransportError subclass: #TTransportClosedError + instanceVariableNames: '' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Transport'! + +Object subclass: #TClient + instanceVariableNames: 'iprot oprot seqid remoteSeqid' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift'! + +!TClient class methodsFor: 'as yet unclassified' stamp: 'pc 11/7/2007 06:00'! +binaryOnHost: aString port: anInteger + | sock | + sock := TSocket new host: aString; port: anInteger; open; yourself. + ^ self new + inProtocol: (TBinaryProtocol new transport: sock); + yourself! ! + +!TClient methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 23:03'! +inProtocol: aProtocol + iprot := aProtocol. + oprot ifNil: [oprot := aProtocol]! ! + +!TClient methodsFor: 'as yet unclassified' stamp: 'pc 10/26/2007 04:28'! +nextSeqid + ^ seqid + ifNil: [seqid := 0] + ifNotNil: [seqid := seqid + 1]! ! + +!TClient methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 22:51'! +outProtocol: aProtocol + oprot := aProtocol! ! + +!TClient methodsFor: 'as yet unclassified' stamp: 'pc 10/28/2007 15:32'! +validateRemoteMessage: aMsg + remoteSeqid + ifNil: [remoteSeqid := aMsg seqid] + ifNotNil: + [(remoteSeqid + 1) = aMsg seqid ifFalse: + [TProtocolError signal: 'Bad seqid: ', aMsg seqid asString, + '; wanted: ', remoteSeqid asString]. + remoteSeqid := aMsg seqid]! ! + +Object subclass: #TField + instanceVariableNames: 'name type id' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Protocol'! + +!TField methodsFor: 'accessing' stamp: 'pc 10/24/2007 20:05'! +id + ^ id ifNil: [0]! ! + +!TField methodsFor: 'accessing' stamp: 'pc 10/24/2007 19:44'! +id: anInteger + id := anInteger! ! + +!TField methodsFor: 'accessing' stamp: 'pc 10/24/2007 20:04'! +name + ^ name ifNil: ['']! ! + +!TField methodsFor: 'accessing' stamp: 'pc 10/24/2007 19:44'! +name: anObject + name := anObject! ! + +!TField methodsFor: 'accessing' stamp: 'pc 10/24/2007 20:05'! +type + ^ type ifNil: [TType stop]! ! + +!TField methodsFor: 'accessing' stamp: 'pc 10/24/2007 19:44'! +type: anInteger + type := anInteger! ! + +Object subclass: #TMessage + instanceVariableNames: 'name seqid type' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Protocol'! + +TMessage subclass: #TCallMessage + instanceVariableNames: '' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Protocol'! + +!TCallMessage methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 22:53'! +type + ^ 1! ! + +!TMessage methodsFor: 'accessing' stamp: 'pc 10/24/2007 20:05'! +name + ^ name ifNil: ['']! ! + +!TMessage methodsFor: 'accessing' stamp: 'pc 10/24/2007 19:35'! +name: aString + name := aString! ! + +!TMessage methodsFor: 'accessing' stamp: 'pc 10/24/2007 20:05'! +seqid + ^ seqid ifNil: [0]! ! + +!TMessage methodsFor: 'accessing' stamp: 'pc 10/24/2007 19:35'! +seqid: anInteger + seqid := anInteger! ! + +!TMessage methodsFor: 'accessing' stamp: 'pc 10/24/2007 20:06'! +type + ^ type ifNil: [0]! ! + +!TMessage methodsFor: 'accessing' stamp: 'pc 10/24/2007 19:35'! +type: anInteger + type := anInteger! ! + +Object subclass: #TProtocol + instanceVariableNames: 'transport' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Protocol'! + +TProtocol subclass: #TBinaryProtocol + instanceVariableNames: '' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Protocol'! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 11/1/2007 04:24'! +intFromByteArray: buf + | vals | + vals := Array new: buf size. + 1 to: buf size do: [:n | vals at: n put: ((buf at: n) bitShift: (buf size - n) * 8)]. + ^ vals sum! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 18:46'! +readBool + ^ self readByte isZero not! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 10/25/2007 00:02'! +readByte + ^ (self transport read: 1) first! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 10/28/2007 16:24'! +readDouble + | val | + val := Float new: 2. + ^ val basicAt: 1 put: (self readRawInt: 4); + basicAt: 2 put: (self readRawInt: 4); + yourself! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 20:02'! +readFieldBegin + | field | + field := TField new type: self readByte. + + ^ field type = TType stop + ifTrue: [field] + ifFalse: [field id: self readI16; yourself]! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:15'! +readI16 + ^ self readInt: 2! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:20'! +readI32 + ^ self readInt: 4! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:20'! +readI64 + ^ self readInt: 8! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 11/1/2007 02:35'! +readInt: size + | buf val | + buf := transport read: size. + val := self intFromByteArray: buf. + ^ buf first > 16r7F + ifTrue: [self unsignedInt: val size: size] + ifFalse: [val]! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:57'! +readListBegin + ^ TList new + elemType: self readByte; + size: self readI32! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:58'! +readMapBegin + ^ TMap new + keyType: self readByte; + valueType: self readByte; + size: self readI32! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 11/1/2007 04:22'! +readMessageBegin + | version | + version := self readI32. + + (version bitAnd: self versionMask) = self version1 + ifFalse: [TProtocolError signalWithCode: TProtocolError badVersion]. + + ^ TMessage new + type: (version bitAnd: 16r000000FF); + name: self readString; + seqid: self readI32! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 10/28/2007 16:24'! +readRawInt: size + ^ self intFromByteArray: (transport read: size)! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 11/1/2007 00:59'! +readSetBegin + "element type, size" + ^ TSet new + elemType: self readByte; + size: self readI32! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 02/07/2009 19:00'! +readString +readString + | sz | + sz := self readI32. + ^ sz > 0 ifTrue: [(transport read: sz) asString] ifFalse: ['']! ! + +!TBinaryProtocol methodsFor: 'reading' stamp: 'pc 11/1/2007 04:22'! +unsignedInt: val size: size + ^ 0 - ((val - 1) bitXor: ((2 raisedTo: (size * 8)) - 1))! ! + +!TBinaryProtocol methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 22:13'! +version1 + ^ 16r80010000 ! ! + +!TBinaryProtocol methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 18:01'! +versionMask + ^ 16rFFFF0000! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 18:35'! +write: aString + transport write: aString! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:23'! +writeBool: bool + bool ifTrue: [self writeByte: 1] + ifFalse: [self writeByte: 0]! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 10/26/2007 09:31'! +writeByte: aNumber + aNumber > 16rFF ifTrue: [TError signal: 'writeByte too big']. + transport write: (Array with: aNumber)! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 10/28/2007 16:16'! +writeDouble: aDouble + self writeI32: (aDouble basicAt: 1); + writeI32: (aDouble basicAt: 2)! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:56'! +writeField: aField + self writeByte: aField type; + writeI16: aField id! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 10/25/2007 00:01'! +writeFieldBegin: aField + self writeByte: aField type. + self writeI16: aField id! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 18:04'! +writeFieldStop + self writeByte: TType stop! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 11/1/2007 02:06'! +writeI16: i16 + self writeInt: i16 size: 2! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 11/1/2007 02:06'! +writeI32: i32 + self writeInt: i32 size: 4! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 11/1/2007 02:06'! +writeI64: i64 + self writeInt: i64 size: 8! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 11/1/2007 04:23'! +writeInt: val size: size + 1 to: size do: [:n | self writeByte: ((val bitShift: (size negated + n) * 8) bitAnd: 16rFF)]! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 11/1/2007 00:48'! +writeListBegin: aList + self writeByte: aList elemType; writeI32: aList size! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:55'! +writeMapBegin: aMap + self writeByte: aMap keyType; + writeByte: aMap valueType; + writeI32: aMap size! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 20:36'! +writeMessageBegin: msg + self writeI32: (self version1 bitOr: msg type); + writeString: msg name; + writeI32: msg seqid! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 11/1/2007 00:56'! +writeSetBegin: aSet + self writeByte: aSet elemType; writeI32: aSet size! ! + +!TBinaryProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 18:35'! +writeString: aString + self writeI32: aString size; + write: aString! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readBool! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readByte! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readDouble! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readFieldBegin! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readFieldEnd! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readI16! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readI32! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readI64! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readListBegin! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readListEnd! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readMapBegin! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readMapEnd! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:39'! +readMessageBegin! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:39'! +readMessageEnd! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readSetBegin! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readSetEnd! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/25/2007 16:10'! +readSimpleType: aType + aType = TType bool ifTrue: [^ self readBool]. + aType = TType byte ifTrue: [^ self readByte]. + aType = TType double ifTrue: [^ self readDouble]. + aType = TType i16 ifTrue: [^ self readI16]. + aType = TType i32 ifTrue: [^ self readI32]. + aType = TType i64 ifTrue: [^ self readI64]. + aType = TType list ifTrue: [^ self readBool].! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readString! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readStructBegin + ! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/24/2007 19:40'! +readStructEnd! ! + +!TProtocol methodsFor: 'reading' stamp: 'pc 10/26/2007 21:34'! +skip: aType + aType = TType stop ifTrue: [^ self]. + aType = TType bool ifTrue: [^ self readBool]. + aType = TType byte ifTrue: [^ self readByte]. + aType = TType i16 ifTrue: [^ self readI16]. + aType = TType i32 ifTrue: [^ self readI32]. + aType = TType i64 ifTrue: [^ self readI64]. + aType = TType string ifTrue: [^ self readString]. + aType = TType double ifTrue: [^ self readDouble]. + aType = TType struct ifTrue: + [| field | + self readStructBegin. + [(field := self readFieldBegin) type = TType stop] whileFalse: + [self skip: field type. self readFieldEnd]. + ^ self readStructEnd]. + aType = TType map ifTrue: + [| map | + map := self readMapBegin. + map size timesRepeat: [self skip: map keyType. self skip: map valueType]. + ^ self readMapEnd]. + aType = TType list ifTrue: + [| list | + list := self readListBegin. + list size timesRepeat: [self skip: list elemType]. + ^ self readListEnd]. + aType = TType set ifTrue: + [| set | + set := self readSetBegin. + set size timesRepeat: [self skip: set elemType]. + ^ self readSetEnd]. + + self error: 'Unknown type'! ! + +!TProtocol methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 23:02'! +transport + ^ transport! ! + +!TProtocol methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:28'! +transport: aTransport + transport := aTransport! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:37'! +writeBool: aBool! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:37'! +writeByte: aByte! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:38'! +writeDouble: aFloat! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:38'! +writeFieldBegin: aField! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:37'! +writeFieldEnd! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:37'! +writeFieldStop! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:37'! +writeI16: i16! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:37'! +writeI32: i32! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:37'! +writeI64: i64! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:39'! +writeListBegin: aList! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:37'! +writeListEnd! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:39'! +writeMapBegin: aMap! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:37'! +writeMapEnd! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:36'! +writeMessageBegin! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:36'! +writeMessageEnd! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:39'! +writeSetBegin: aSet! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:37'! +writeSetEnd! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:38'! +writeString: aString! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:38'! +writeStructBegin: aStruct! ! + +!TProtocol methodsFor: 'writing' stamp: 'pc 10/24/2007 19:37'! +writeStructEnd! ! + +Object subclass: #TResult + instanceVariableNames: 'success oprot iprot exception' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift'! + +!TResult methodsFor: 'as yet unclassified' stamp: 'pc 10/26/2007 21:35'! +exception + ^ exception! ! + +!TResult methodsFor: 'as yet unclassified' stamp: 'pc 10/26/2007 21:35'! +exception: anError + exception := anError! ! + +!TResult methodsFor: 'as yet unclassified' stamp: 'pc 10/26/2007 14:43'! +success + ^ success! ! + +!TResult methodsFor: 'as yet unclassified' stamp: 'pc 10/26/2007 14:43'! +success: anObject + success := anObject! ! + +Object subclass: #TSizedObject + instanceVariableNames: 'size' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Protocol'! + +TSizedObject subclass: #TList + instanceVariableNames: 'elemType' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Protocol'! + +!TList methodsFor: 'accessing' stamp: 'pc 10/24/2007 20:04'! +elemType + ^ elemType ifNil: [TType stop]! ! + +!TList methodsFor: 'accessing' stamp: 'pc 10/24/2007 19:42'! +elemType: anInteger + elemType := anInteger! ! + +TList subclass: #TSet + instanceVariableNames: '' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Protocol'! + +TSizedObject subclass: #TMap + instanceVariableNames: 'keyType valueType' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Protocol'! + +!TMap methodsFor: 'accessing' stamp: 'pc 10/24/2007 20:04'! +keyType + ^ keyType ifNil: [TType stop]! ! + +!TMap methodsFor: 'accessing' stamp: 'pc 10/24/2007 19:45'! +keyType: anInteger + keyType := anInteger! ! + +!TMap methodsFor: 'accessing' stamp: 'pc 10/24/2007 20:04'! +valueType + ^ valueType ifNil: [TType stop]! ! + +!TMap methodsFor: 'accessing' stamp: 'pc 10/24/2007 19:45'! +valueType: anInteger + valueType := anInteger! ! + +!TSizedObject methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 20:03'! +size + ^ size ifNil: [0]! ! + +!TSizedObject methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 20:06'! +size: anInteger + size := anInteger! ! + +Object subclass: #TSocket + instanceVariableNames: 'host port stream' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Transport'! + +!TSocket methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 22:34'! +close + self isOpen ifTrue: [stream close]! ! + +!TSocket methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 22:23'! +connect + ^ (self socketStream openConnectionToHost: + (NetNameResolver addressForName: host) port: port) + timeout: 180; + binary; + yourself! ! + +!TSocket methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 20:35'! +flush + stream flush! ! + +!TSocket methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:08'! +host: aString + host := aString! ! + +!TSocket methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 20:34'! +isOpen + ^ stream isNil not + and: [stream socket isConnected] + and: [stream socket isOtherEndClosed not]! ! + +!TSocket methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 22:22'! +open + stream := self connect! ! + +!TSocket methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:09'! +port: anInteger + port := anInteger! ! + +!TSocket methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:17'! +read: size + | data | + [data := stream next: size. + data isEmpty ifTrue: [TTransportError signal: 'Could not read ', size asString, ' bytes']. + ^ data] + on: ConnectionClosed + do: [TTransportClosedError signal]! ! + +!TSocket methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 22:18'! +socketStream + ^ Smalltalk at: #FastSocketStream ifAbsent: [SocketStream] ! ! + +!TSocket methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 22:17'! +write: aCollection + [stream nextPutAll: aCollection] + on: ConnectionClosed + do: [TTransportClosedError signal]! ! + +Object subclass: #TStruct + instanceVariableNames: 'name' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Protocol'! + +!TStruct methodsFor: 'accessing' stamp: 'pc 10/24/2007 19:47'! +name + ^ name! ! + +!TStruct methodsFor: 'accessing' stamp: 'pc 10/24/2007 19:47'! +name: aString + name := aString! ! + +Object subclass: #TTransport + instanceVariableNames: '' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift-Transport'! + +!TTransport methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:18'! +close + self subclassResponsibility! ! + +!TTransport methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:22'! +flush + self subclassResponsibility! ! + +!TTransport methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:18'! +isOpen + self subclassResponsibility! ! + +!TTransport methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:18'! +open + self subclassResponsibility! ! + +!TTransport methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:18'! +read: anInteger + self subclassResponsibility! ! + +!TTransport methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:22'! +readAll: anInteger + ^ String streamContents: [:str | + [str size < anInteger] whileTrue: + [str nextPutAll: (self read: anInteger - str size)]]! ! + +!TTransport methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:22'! +write: aString + self subclassResponsibility! ! + +Object subclass: #TType + instanceVariableNames: '' + classVariableNames: '' + poolDictionaries: '' + category: 'Thrift'! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:03'! +bool + ^ 2! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:03'! +byte + ^ 3! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/25/2007 15:55'! +codeOf: aTypeName + self typeMap do: [:each | each first = aTypeName ifTrue: [^ each second]]. + ^ nil! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:03'! +double + ^ 4! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:04'! +i16 + ^ 6! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:04'! +i32 + ^ 8! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:04'! +i64 + ^ 10! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:04'! +list + ^ 15! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:04'! +map + ^ 13! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/25/2007 15:56'! +nameOf: aTypeCode + self typeMap do: [:each | each second = aTypeCode ifTrue: [^ each first]]. + ^ nil! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:04'! +set + ^ 14! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:03'! +stop + ^ 0! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:04'! +string + ^ 11! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:04'! +struct + ^ 12! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/25/2007 15:51'! +typeMap + ^ #((bool 2) (byte 3) (double 4) (i16 6) (i32 8) (i64 10) (list 15) + (map 13) (set 15) (stop 0) (string 11) (struct 12) (void 1))! ! + +!TType class methodsFor: 'as yet unclassified' stamp: 'pc 10/24/2007 17:03'! +void + ^ 1! ! diff --git a/print_version.sh b/print_version.sh new file mode 100755 index 00000000..f36fa5ea --- /dev/null +++ b/print_version.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +cd "`dirname "$0"`" + +# Computing both the version and the revision on every invocation is wasteful, +# but it is cheap and avoids the use of nonportable shell functions. + +VERSION=`sed -ne 's/^AC_INIT(\[thrift\], \[\(.*\)\])$/\1/p' configure.ac` + +if test -d .svn ; then + REVISION="r`svnversion`" +elif test -d .git ; then + SHA1=`git rev-list --max-count=1 --grep='^git-svn-id:' HEAD` + REVISION=`git cat-file commit $SHA1 | sed -ne 's/^git-svn-id:[^@]*@\([0-9][0-9]*\).*/r\1/p'` + OFFSET=`git rev-list ^$SHA1 HEAD | wc -l | tr -d ' '` + if test $OFFSET != 0 ; then + REVISION="$REVISION-$OFFSET-`git rev-parse --verify HEAD | cut -c 1-7`" + fi +else + REVISION="exported" +fi + +case "$1" in + -v) echo $VERSION ;; + -r) echo $REVISION ;; + -a) echo "$VERSION-$REVISION" ;; + *) echo "Usage: $0 -v|-r|-a"; exit 1;; +esac diff --git a/test/AllProtocolTests.cpp b/test/AllProtocolTests.cpp new file mode 100644 index 00000000..db29cccf --- /dev/null +++ b/test/AllProtocolTests.cpp @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include "AllProtocolTests.tcc" + +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; + +char errorMessage[ERR_LEN]; + +int main(int argc, char** argv) { + try { + testProtocol("TBinaryProtocol"); + testProtocol("TCompactProtocol"); + } catch (TException e) { + printf("%s\n", e.what()); + return 1; + } + return 0; +} diff --git a/test/AllProtocolTests.tcc b/test/AllProtocolTests.tcc new file mode 100644 index 00000000..a5a31156 --- /dev/null +++ b/test/AllProtocolTests.tcc @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TEST_GENERICPROTOCOLTEST_TCC_ +#define _THRIFT_TEST_GENERICPROTOCOLTEST_TCC_ 1 + +#include + +#include +#include +#include + +#include "GenericHelpers.h" + +using boost::shared_ptr; +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; + +#define ERR_LEN 512 +extern char errorMessage[ERR_LEN]; + +template +void testNaked(Val val) { + shared_ptr transport(new TMemoryBuffer()); + shared_ptr protocol(new TProto(transport)); + + GenericIO::write(protocol, val); + Val out; + GenericIO::read(protocol, out); + if (out != val) { + snprintf(errorMessage, ERR_LEN, "Invalid naked test (type: %s)", ClassNames::getName()); + throw TException(errorMessage); + } +} + +template +void testField(const Val val) { + shared_ptr transport(new TMemoryBuffer()); + shared_ptr protocol(new TProto(transport)); + + protocol->writeStructBegin("test_struct"); + protocol->writeFieldBegin("test_field", type, (int16_t)15); + + GenericIO::write(protocol, val); + + protocol->writeFieldEnd(); + protocol->writeStructEnd(); + + std::string name; + TType fieldType; + int16_t fieldId; + + protocol->readStructBegin(name); + protocol->readFieldBegin(name, fieldType, fieldId); + + if (fieldId != 15) { + snprintf(errorMessage, ERR_LEN, "Invalid ID (type: %s)", typeid(val).name()); + throw TException(errorMessage); + } + if (fieldType != type) { + snprintf(errorMessage, ERR_LEN, "Invalid Field Type (type: %s)", typeid(val).name()); + throw TException(errorMessage); + } + + Val out; + GenericIO::read(protocol, out); + + if (out != val) { + snprintf(errorMessage, ERR_LEN, "Invalid value read (type: %s)", typeid(val).name()); + throw TException(errorMessage); + } + + protocol->readFieldEnd(); + protocol->readStructEnd(); +} + +template +void testMessage() { + struct TMessage { + const char* name; + TMessageType type; + int32_t seqid; + } messages[4] = { + {"short message name", T_CALL, 0}, + {"1", T_REPLY, 12345}, + {"loooooooooooooooooooooooooooooooooong", T_EXCEPTION, 1 << 16}, + {"Janky", T_CALL, 0} + }; + + for (int i = 0; i < 4; i++) { + shared_ptr transport(new TMemoryBuffer()); + shared_ptr protocol(new TProto(transport)); + + protocol->writeMessageBegin(messages[i].name, + messages[i].type, + messages[i].seqid); + protocol->writeMessageEnd(); + + std::string name; + TMessageType type; + int32_t seqid; + + protocol->readMessageBegin(name, type, seqid); + if (name != messages[i].name || + type != messages[i].type || + seqid != messages[i].seqid) { + throw TException("readMessageBegin failed."); + } + } +} + +template +void testProtocol(const char* protoname) { + try { + testNaked((int8_t)123); + + for (int32_t i = 0; i < 128; i++) { + testField((int8_t)i); + testField((int8_t)-i); + } + + testNaked((int16_t)0); + testNaked((int16_t)1); + testNaked((int16_t)15000); + testNaked((int16_t)0x7fff); + testNaked((int16_t)-1); + testNaked((int16_t)-15000); + testNaked((int16_t)-0x7fff); + testNaked(std::numeric_limits::min()); + testNaked(std::numeric_limits::max()); + + testField((int16_t)0); + testField((int16_t)1); + testField((int16_t)7); + testField((int16_t)150); + testField((int16_t)15000); + testField((int16_t)0x7fff); + testField((int16_t)-1); + testField((int16_t)-7); + testField((int16_t)-150); + testField((int16_t)-15000); + testField((int16_t)-0x7fff); + + testNaked(0); + testNaked(1); + testNaked(15000); + testNaked(0xffff); + testNaked(-1); + testNaked(-15000); + testNaked(-0xffff); + testNaked(std::numeric_limits::min()); + testNaked(std::numeric_limits::max()); + + testField(0); + testField(1); + testField(7); + testField(150); + testField(15000); + testField(31337); + testField(0xffff); + testField(0xffffff); + testField(-1); + testField(-7); + testField(-150); + testField(-15000); + testField(-0xffff); + testField(-0xffffff); + testNaked(std::numeric_limits::min()); + testNaked(std::numeric_limits::max()); + testNaked(std::numeric_limits::min() + 10); + testNaked(std::numeric_limits::max() - 16); + testNaked(std::numeric_limits::min()); + testNaked(std::numeric_limits::max()); + + + testNaked(0); + for (int64_t i = 0; i < 62; i++) { + testNaked(1L << i); + testNaked(-(1L << i)); + } + + testField(0); + for (int i = 0; i < 62; i++) { + testField(1L << i); + testField(-(1L << i)); + } + + testNaked(123.456); + + testNaked(""); + testNaked("short"); + testNaked("borderlinetiny"); + testNaked("a bit longer than the smallest possible"); + testNaked("\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA"); //kinda binary test + + testField(""); + testField("short"); + testField("borderlinetiny"); + testField("a bit longer than the smallest possible"); + + testMessage(); + + printf("%s => OK\n", protoname); + } catch (TException e) { + snprintf(errorMessage, ERR_LEN, "%s => Test FAILED: %s", protoname, e.what()); + throw TException(errorMessage); + } +} + +#endif diff --git a/test/AnnotationTest.thrift b/test/AnnotationTest.thrift new file mode 100644 index 00000000..dcc41b0b --- /dev/null +++ b/test/AnnotationTest.thrift @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +typedef list ( cpp.template = "std::list" ) int_linked_list + +struct foo { + 1: i32 bar; + 2: i32 baz; + 3: i32 qux; + 4: i32 bop; +} ( + cpp.type = "DenseFoo", + python.type = "DenseFoo", + java.final = "", +) diff --git a/test/Benchmark.cpp b/test/Benchmark.cpp new file mode 100644 index 00000000..d315fca7 --- /dev/null +++ b/test/Benchmark.cpp @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include "gen-cpp/DebugProtoTest_types.h" +#include +#include "../lib/cpp/src/protocol/TDebugProtocol.h" +#include + +class Timer { +public: + timeval vStart; + + Timer() { + gettimeofday(&vStart, 0); + } + void start() { + gettimeofday(&vStart, 0); + } + + double frame() { + timeval vEnd; + gettimeofday(&vEnd, 0); + double dstart = vStart.tv_sec + ((double)vStart.tv_usec / 1000000.0); + double dend = vEnd.tv_sec + ((double)vEnd.tv_usec / 1000000.0); + return dend - dstart; + } + +}; + +int main() { + using namespace std; + using namespace thrift::test::debug; + using namespace apache::thrift::transport; + using namespace apache::thrift::protocol; + using namespace boost; + + OneOfEach ooe; + ooe.im_true = true; + ooe.im_false = false; + ooe.a_bite = 0xd6; + ooe.integer16 = 27000; + ooe.integer32 = 1<<24; + ooe.integer64 = (uint64_t)6000 * 1000 * 1000; + ooe.double_precision = M_PI; + ooe.some_characters = "JSON THIS! \"\1"; + ooe.zomg_unicode = "\xd7\n\a\t"; + ooe.base64 = "\1\2\3\255"; + + shared_ptr buf(new TMemoryBuffer()); + + int num = 1000000; + + { + Timer timer; + + for (int i = 0; i < num; i ++) { + buf->resetBuffer(); + TBinaryProtocol prot(buf); + ooe.write(&prot); + } + cout << "Write: " << num / (1000 * timer.frame()) << " kHz" << endl; + } + + uint8_t* data; + uint32_t datasize; + + buf->getBuffer(&data, &datasize); + + { + + Timer timer; + + for (int i = 0; i < num; i ++) { + OneOfEach ooe2; + shared_ptr buf2(new TMemoryBuffer(data, datasize)); + //buf2->resetBuffer(data, datasize); + TBinaryProtocol prot(buf2); + ooe2.read(&prot); + + //cout << apache::thrift::ThriftDebugString(ooe2) << endl << endl; + } + cout << " Read: " << num / (1000 * timer.frame()) << " kHz" << endl; + } + + + return 0; +} diff --git a/test/BrokenConstants.thrift b/test/BrokenConstants.thrift new file mode 100644 index 00000000..c5aab4ab --- /dev/null +++ b/test/BrokenConstants.thrift @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const i64 myint = 68719476736 +const i64 broken = 9876543210987654321 // A little over 2^63 + +enum foo { + bar = 68719476736 +} diff --git a/test/ConstantsDemo.thrift b/test/ConstantsDemo.thrift new file mode 100644 index 00000000..7e97f02c --- /dev/null +++ b/test/ConstantsDemo.thrift @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace cpp yozone + +struct thing { + 1: i32 hello, + 2: i32 goodbye +} + +enum enumconstants { + ONE = 1, + TWO = 2 +} + +struct thing2 { + 1: enumconstants val = TWO +} + +typedef i32 myIntType +const myIntType myInt = 3 + +const map GEN_ENUM_NAMES = {ONE : "HOWDY", TWO: PARTNER} + +const i32 hex_const = 0x0001F + +const i32 GEN_ME = -3523553 +const double GEn_DUB = 325.532 +const double GEn_DU = 085.2355 +const string GEN_STRING = "asldkjasfd" + +const map GEN_MAP = { 35532 : 233, 43523 : 853 } +const list GEN_LIST = [ 235235, 23598352, 3253523 ] + +const map> GEN_MAPMAP = { 235 : { 532 : 53255, 235:235}} + +const map GEN_MAP2 = { "hello" : 233, "lkj98d" : 853, 'lkjsdf' : 098325 } + +const thing GEN_THING = { 'hello' : 325, 'goodbye' : 325352 } + +const map GEN_WHAT = { 35 : { 'hello' : 325, 'goodbye' : 325352 } } + +const set GEN_SET = [ 235, 235, 53235 ] + +exception Blah { + 1: i32 bing } + +exception Gak {} + +service yowza { + void blingity(), + i32 blangity() throws (1: Blah hoot ) +} diff --git a/test/DebugProtoTest.cpp b/test/DebugProtoTest.cpp new file mode 100644 index 00000000..8c9aabec --- /dev/null +++ b/test/DebugProtoTest.cpp @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include "gen-cpp/DebugProtoTest_types.h" +#include "../lib/cpp/src/protocol/TDebugProtocol.h" + +int main() { + using std::cout; + using std::endl; + using namespace thrift::test::debug; + + + OneOfEach ooe; + ooe.im_true = true; + ooe.im_false = false; + ooe.a_bite = 0xd6; + ooe.integer16 = 27000; + ooe.integer32 = 1<<24; + ooe.integer64 = (uint64_t)6000 * 1000 * 1000; + ooe.double_precision = M_PI; + ooe.some_characters = "Debug THIS!"; + ooe.zomg_unicode = "\xd7\n\a\t"; + + cout << apache::thrift::ThriftDebugString(ooe) << endl << endl; + + + Nesting n; + n.my_ooe = ooe; + n.my_ooe.integer16 = 16; + n.my_ooe.integer32 = 32; + n.my_ooe.integer64 = 64; + n.my_ooe.double_precision = (std::sqrt(5)+1)/2; + n.my_ooe.some_characters = ":R (me going \"rrrr\")"; + n.my_ooe.zomg_unicode = "\xd3\x80\xe2\x85\xae\xce\x9d\x20" + "\xd0\x9d\xce\xbf\xe2\x85\xbf\xd0\xbe\xc9\xa1\xd0\xb3\xd0\xb0\xcf\x81\xe2\x84\x8e" + "\x20\xce\x91\x74\x74\xce\xb1\xe2\x85\xbd\xce\xba\xc7\x83\xe2\x80\xbc"; + n.my_bonk.type = 31337; + n.my_bonk.message = "I am a bonk... xor!"; + + cout << apache::thrift::ThriftDebugString(n) << endl << endl; + + + HolyMoley hm; + + hm.big.push_back(ooe); + hm.big.push_back(n.my_ooe); + hm.big[0].a_bite = 0x22; + hm.big[1].a_bite = 0x33; + + std::vector stage1; + stage1.push_back("and a one"); + stage1.push_back("and a two"); + hm.contain.insert(stage1); + stage1.clear(); + stage1.push_back("then a one, two"); + stage1.push_back("three!"); + stage1.push_back("FOUR!!"); + hm.contain.insert(stage1); + stage1.clear(); + hm.contain.insert(stage1); + + std::vector stage2; + hm.bonks["nothing"] = stage2; + stage2.resize(stage2.size()+1); + stage2.back().type = 1; + stage2.back().message = "Wait."; + stage2.resize(stage2.size()+1); + stage2.back().type = 2; + stage2.back().message = "What?"; + hm.bonks["something"] = stage2; + stage2.clear(); + stage2.resize(stage2.size()+1); + stage2.back().type = 3; + stage2.back().message = "quoth"; + stage2.resize(stage2.size()+1); + stage2.back().type = 4; + stage2.back().message = "the raven"; + stage2.resize(stage2.size()+1); + stage2.back().type = 5; + stage2.back().message = "nevermore"; + hm.bonks["poe"] = stage2; + + cout << apache::thrift::ThriftDebugString(hm) << endl << endl; + + + return 0; +} diff --git a/test/DebugProtoTest.thrift b/test/DebugProtoTest.thrift new file mode 100644 index 00000000..d3d25802 --- /dev/null +++ b/test/DebugProtoTest.thrift @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace cpp thrift.test.debug +namespace java thrift.test + +struct Doubles { + 1: double nan, + 2: double inf, + 3: double neginf, + 4: double repeating, + 5: double big, + 6: double small, + 7: double zero, + 8: double negzero, +} + +struct OneOfEach { + 1: bool im_true, + 2: bool im_false, + 3: byte a_bite = 200, + 4: i16 integer16 = 33000, + 5: i32 integer32, + 6: i64 integer64 = 10000000000, + 7: double double_precision, + 8: string some_characters, + 9: string zomg_unicode, + 10: bool what_who, + 11: binary base64, + 12: list byte_list = [1, 2, 3], + 13: list i16_list = [1,2,3], + 14: list i64_list = [1,2,3] +} + +struct Bonk { + 1: i32 type, + 2: string message, +} + +struct Nesting { + 1: Bonk my_bonk, + 2: OneOfEach my_ooe, +} + +struct HolyMoley { + 1: list big, + 2: set> contain, + 3: map> bonks, +} + +struct Backwards { + 2: i32 first_tag2, + 1: i32 second_tag1, +} + +struct Empty { +} + +struct Wrapper { + 1: Empty foo +} + +struct RandomStuff { + 1: i32 a, + 2: i32 b, + 3: i32 c, + 4: i32 d, + 5: list myintlist, + 6: map maps, + 7: i64 bigint, + 8: double triple, +} + +struct Base64 { + 1: i32 a, + 2: binary b1, + 3: binary b2, + 4: binary b3, + 5: binary b4, + 6: binary b5, + 7: binary b6, +} + +struct CompactProtoTestStruct { + // primitive fields + 1: byte a_byte; + 2: i16 a_i16; + 3: i32 a_i32; + 4: i64 a_i64; + 5: double a_double; + 6: string a_string; + 7: binary a_binary; + 8: bool true_field; + 9: bool false_field; + 10: Empty empty_struct_field; + + // primitives in lists + 11: list byte_list; + 12: list i16_list; + 13: list i32_list; + 14: list i64_list; + 15: list double_list; + 16: list string_list; + 17: list binary_list; + 18: list boolean_list; + 19: list struct_list; + + // primitives in sets + 20: set byte_set; + 21: set i16_set; + 22: set i32_set; + 23: set i64_set; + 24: set double_set; + 25: set string_set; + 26: set binary_set; + 27: set boolean_set; + 28: set struct_set; + + // maps + // primitives as keys + 29: map byte_byte_map; + 30: map i16_byte_map; + 31: map i32_byte_map; + 32: map i64_byte_map; + 33: map double_byte_map; + 34: map string_byte_map; + 35: map binary_byte_map; + 36: map boolean_byte_map; + // primitives as values + 37: map byte_i16_map; + 38: map byte_i32_map; + 39: map byte_i64_map; + 40: map byte_double_map; + 41: map byte_string_map; + 42: map byte_binary_map; + 43: map byte_boolean_map; + // collections as keys + 44: map, byte> list_byte_map; + 45: map, byte> set_byte_map; + 46: map, byte> map_byte_map; + // collections as values + 47: map> byte_map_map; + 48: map> byte_set_map; + 49: map> byte_list_map; +} + + +const CompactProtoTestStruct COMPACT_TEST = { + 'a_byte' : 127, + 'a_i16' : 32000, + 'a_i32' : 1000000000, + 'a_i64' : 0xffffffffff, + 'a_double' : 5.6789, + 'a_string' : "my string", +//'a_binary,' + 'true_field' : 1, + 'false_field' : 0, + 'empty_struct_field' : {}, + 'byte_list' : [-127, -1, 0, 1, 127], + 'i16_list' : [-1, 0, 1, 0x7fff], + 'i32_list' : [-1, 0, 0xff, 0xffff, 0xffffff, 0x7fffffff], + 'i64_list' : [-1, 0, 0xff, 0xffff, 0xffffff, 0xffffffff, 0xffffffffff, 0xffffffffffff, 0xffffffffffffff, 0x7fffffffffffffff], + 'double_list' : [0.1, 0.2, 0.3], + 'string_list' : ["first", "second", "third"], +//'binary_list,' + 'boolean_list' : [1, 1, 1, 0, 0, 0], + 'struct_list' : [{}, {}], + 'byte_set' : [-127, -1, 0, 1, 127], + 'i16_set' : [-1, 0, 1, 0x7fff], + 'i32_set' : [1, 2, 3], + 'i64_set' : [-1, 0, 0xff, 0xffff, 0xffffff, 0xffffffff, 0xffffffffff, 0xffffffffffff, 0xffffffffffffff, 0x7fffffffffffffff], + 'double_set' : [0.1, 0.2, 0.3], + 'string_set' : ["first", "second", "third"], +//'binary_set,' + 'boolean_set' : [1, 0], + 'struct_set' : [{}], + 'byte_byte_map' : {1 : 2}, + 'i16_byte_map' : {1 : 1, -1 : 1, 0x7fff : 1}, + 'i32_byte_map' : {1 : 1, -1 : 1, 0x7fffffff : 1}, + 'i64_byte_map' : {0 : 1, 1 : 1, -1 : 1, 0x7fffffffffffffff : 1}, + 'double_byte_map' : {-1.1 : 1, 1.1 : 1}, + 'string_byte_map' : {"first" : 1, "second" : 2, "third" : 3, "" : 0}, +//'binary_byte_map,' + 'boolean_byte_map' : {1 : 1, 0 : 0}, + 'byte_i16_map' : {1 : 1, 2 : -1, 3 : 0x7fff}, + 'byte_i32_map' : {1 : 1, 2 : -1, 3 : 0x7fffffff}, + 'byte_i64_map' : {1 : 1, 2 : -1, 3 : 0x7fffffffffffffff}, + 'byte_double_map' : {1 : 0.1, 2 : -0.1, 3 : 1000000.1}, + 'byte_string_map' : {1 : "", 2 : "blah", 3 : "loooooooooooooong string"}, +//'byte_binary_map,' + 'byte_boolean_map' : {1 : 1, 2 : 0}, + 'list_byte_map' : {[1, 2, 3] : 1, [0, 1] : 2, [] : 0}, + 'set_byte_map' : {[1, 2, 3] : 1, [0, 1] : 2, [] : 0}, + 'map_byte_map' : {{1 : 1} : 1, {2 : 2} : 2, {} : 0}, + 'byte_map_map' : {0 : {}, 1 : {1 : 1}, 2 : {1 : 1, 2 : 2}}, + 'byte_set_map' : {0 : [], 1 : [1], 2 : [1, 2]}, + 'byte_list_map' : {0 : [], 1 : [1], 2 : [1, 2]}, +} + + + +service Srv { + i32 Janky(1: i32 arg); + + // return type only methods + + void voidMethod(); + i32 primitiveMethod(); + CompactProtoTestStruct structMethod(); +} + +service Inherited extends Srv { + i32 identity(1: i32 arg) +} + +service EmptyService {} + +// The only purpose of this thing is to increase the size of the generated code +// so that ZlibTest has more highly compressible data to play with. +struct BlowUp { + 1: map,set>> b1; + 2: map,set>> b2; + 3: map,set>> b3; + 4: map,set>> b4; +} + + +struct ReverseOrderStruct { + 4: string first; + 3: i16 second; + 2: i32 third; + 1: i64 fourth; +} + +service ReverseOrderService { + void myMethod(4: string first, 3: i16 second, 2: i32 third, 1: i64 fourth); +} \ No newline at end of file diff --git a/test/DebugProtoTest_extras.cpp b/test/DebugProtoTest_extras.cpp new file mode 100644 index 00000000..e68c544b --- /dev/null +++ b/test/DebugProtoTest_extras.cpp @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Extra functions required for DebugProtoTest_types to work + +#include "gen-cpp/DebugProtoTest_types.h" + + +namespace thrift { namespace test { namespace debug { + +bool Empty::operator<(Empty const& other) const { + // It is empty, so all are equal. + return false; +} + +}}} diff --git a/test/DenseLinkingTest.thrift b/test/DenseLinkingTest.thrift new file mode 100644 index 00000000..cf61496a --- /dev/null +++ b/test/DenseLinkingTest.thrift @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* +../compiler/cpp/thrift -gen cpp:dense DebugProtoTest.thrift +../compiler/cpp/thrift -gen cpp:dense DenseLinkingTest.thrift +g++ -Wall -g -I../lib/cpp/src -I/usr/local/include/boost-1_33_1 \ + DebugProtoTest.cpp gen-cpp/DebugProtoTest_types.cpp \ + gen-cpp/DenseLinkingTest_types.cpp \ + ../lib/cpp/.libs/libthrift.a -o DebugProtoTest +./DebugProtoTest +*/ + +/* +The idea of this test is that everything is structurally identical to DebugProtoTest. +If I messed up the naming of the reflection local typespecs, +then compiling this should give errors because of doubly defined symbols. +*/ + +namespace cpp thrift.test + +struct OneOfEachZZ { + 1: bool im_true, + 2: bool im_false, + 3: byte a_bite, + 4: i16 integer16, + 5: i32 integer32, + 6: i64 integer64, + 7: double double_precision, + 8: string some_characters, + 9: string zomg_unicode, + 10: bool what_who, +} + +struct BonkZZ { + 1: i32 type, + 2: string message, +} + +struct NestingZZ { + 1: BonkZZ my_bonk, + 2: OneOfEachZZ my_ooe, +} + +struct HolyMoleyZZ { + 1: list big, + 2: set> contain, + 3: map> bonks, +} + +struct BackwardsZZ { + 2: i32 first_tag2, + 1: i32 second_tag1, +} + +struct EmptyZZ { +} + +struct WrapperZZ { + 1: EmptyZZ foo +} + +struct RandomStuffZZ { + 1: i32 a, + 2: i32 b, + 3: i32 c, + 4: i32 d, + 5: list myintlist, + 6: map maps, + 7: i64 bigint, + 8: double triple, +} + +service Srv { + i32 Janky(1: i32 arg) +} diff --git a/test/DenseProtoTest.cpp b/test/DenseProtoTest.cpp new file mode 100644 index 00000000..99f78655 --- /dev/null +++ b/test/DenseProtoTest.cpp @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* +../compiler/cpp/thrift --gen cpp:dense DebugProtoTest.thrift +../compiler/cpp/thrift --gen cpp:dense OptionalRequiredTest.thrift +g++ -Wall -g -I../lib/cpp/src -I/usr/local/include/boost-1_33_1 \ + gen-cpp/OptionalRequiredTest_types.cpp \ + gen-cpp/DebugProtoTest_types.cpp \ + DenseProtoTest.cpp ../lib/cpp/.libs/libthrift.a -o DenseProtoTest +./DenseProtoTest +*/ + +// I do this to reach into the guts of TDenseProtocol. Sorry. +#define private public +#define inline + +#undef NDEBUG +#include +#include +#include +#include +#include +#include "gen-cpp/DebugProtoTest_types.h" +#include "gen-cpp/OptionalRequiredTest_types.h" +#include +#include + + +// Can't use memcmp here. GCC is too smart. +bool my_memeq(const char* str1, const char* str2, int len) { + for (int i = 0; i < len; i++) { + if (str1[i] != str2[i]) { + return false; + } + } + return true; +} + + +int main() { + using std::string; + using std::cout; + using std::endl; + using boost::shared_ptr; + using namespace thrift::test::debug; + using namespace apache::thrift::transport; + using namespace apache::thrift::protocol; + + + OneOfEach ooe; + ooe.im_true = true; + ooe.im_false = false; + ooe.a_bite = 0xd6; + ooe.integer16 = 27000; + ooe.integer32 = 1<<24; + ooe.integer64 = (uint64_t)6000 * 1000 * 1000; + ooe.double_precision = M_PI; + ooe.some_characters = "Debug THIS!"; + ooe.zomg_unicode = "\xd7\n\a\t"; + + //cout << apache::thrift::ThriftDebugString(ooe) << endl << endl; + + + Nesting n; + n.my_ooe = ooe; + n.my_ooe.integer16 = 16; + n.my_ooe.integer32 = 32; + n.my_ooe.integer64 = 64; + n.my_ooe.double_precision = (std::sqrt(5)+1)/2; + n.my_ooe.some_characters = ":R (me going \"rrrr\")"; + n.my_ooe.zomg_unicode = "\xd3\x80\xe2\x85\xae\xce\x9d\x20" + "\xd0\x9d\xce\xbf\xe2\x85\xbf\xd0\xbe\xc9\xa1\xd0\xb3\xd0\xb0\xcf\x81\xe2\x84\x8e" + "\x20\xce\x91\x74\x74\xce\xb1\xe2\x85\xbd\xce\xba\xc7\x83\xe2\x80\xbc"; + n.my_bonk.type = 31337; + n.my_bonk.message = "I am a bonk... xor!"; + + //cout << apache::thrift::ThriftDebugString(n) << endl << endl; + + + HolyMoley hm; + + hm.big.push_back(ooe); + hm.big.push_back(n.my_ooe); + hm.big[0].a_bite = 0x22; + hm.big[1].a_bite = 0x33; + + std::vector stage1; + stage1.push_back("and a one"); + stage1.push_back("and a two"); + hm.contain.insert(stage1); + stage1.clear(); + stage1.push_back("then a one, two"); + stage1.push_back("three!"); + stage1.push_back("FOUR!!"); + hm.contain.insert(stage1); + stage1.clear(); + hm.contain.insert(stage1); + + std::vector stage2; + hm.bonks["nothing"] = stage2; + stage2.resize(stage2.size()+1); + stage2.back().type = 1; + stage2.back().message = "Wait."; + stage2.resize(stage2.size()+1); + stage2.back().type = 2; + stage2.back().message = "What?"; + hm.bonks["something"] = stage2; + stage2.clear(); + stage2.resize(stage2.size()+1); + stage2.back().type = 3; + stage2.back().message = "quoth"; + stage2.resize(stage2.size()+1); + stage2.back().type = 4; + stage2.back().message = "the raven"; + stage2.resize(stage2.size()+1); + stage2.back().type = 5; + stage2.back().message = "nevermore"; + hm.bonks["poe"] = stage2; + + //cout << apache::thrift::ThriftDebugString(hm) << endl << endl; + + shared_ptr buffer(new TMemoryBuffer()); + shared_ptr proto(new TDenseProtocol(buffer)); + proto->setTypeSpec(HolyMoley::local_reflection); + + hm.write(proto.get()); + HolyMoley hm2; + hm2.read(proto.get()); + + assert(hm == hm2); + + + // Let's test out the variable-length ints, shall we? + uint64_t vlq; + #define checkout(i, c) { \ + buffer->resetBuffer(); \ + proto->vlqWrite(i); \ + proto->getTransport()->flush(); \ + assert(my_memeq(buffer->getBufferAsString().data(), c, sizeof(c)-1)); \ + proto->vlqRead(vlq); \ + assert(vlq == i); \ + } + + checkout(0x00000000, "\x00"); + checkout(0x00000040, "\x40"); + checkout(0x0000007F, "\x7F"); + checkout(0x00000080, "\x81\x00"); + checkout(0x00002000, "\xC0\x00"); + checkout(0x00003FFF, "\xFF\x7F"); + checkout(0x00004000, "\x81\x80\x00"); + checkout(0x00100000, "\xC0\x80\x00"); + checkout(0x001FFFFF, "\xFF\xFF\x7F"); + checkout(0x00200000, "\x81\x80\x80\x00"); + checkout(0x08000000, "\xC0\x80\x80\x00"); + checkout(0x0FFFFFFF, "\xFF\xFF\xFF\x7F"); + checkout(0x10000000, "\x81\x80\x80\x80\x00"); + checkout(0x20000000, "\x82\x80\x80\x80\x00"); + checkout(0x1FFFFFFF, "\x81\xFF\xFF\xFF\x7F"); + checkout(0xFFFFFFFF, "\x8F\xFF\xFF\xFF\x7F"); + + checkout(0x0000000100000000ull, "\x90\x80\x80\x80\x00"); + checkout(0x0000000200000000ull, "\xA0\x80\x80\x80\x00"); + checkout(0x0000000300000000ull, "\xB0\x80\x80\x80\x00"); + checkout(0x0000000700000000ull, "\xF0\x80\x80\x80\x00"); + checkout(0x00000007F0000000ull, "\xFF\x80\x80\x80\x00"); + checkout(0x00000007FFFFFFFFull, "\xFF\xFF\xFF\xFF\x7F"); + checkout(0x0000000800000000ull, "\x81\x80\x80\x80\x80\x00"); + checkout(0x1FFFFFFFFFFFFFFFull, "\x9F\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F"); + checkout(0x7FFFFFFFFFFFFFFFull, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F"); + checkout(0xFFFFFFFFFFFFFFFFull, "\x81\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F"); + + // Test out the slow path with a TBufferedTransport. + shared_ptr buff_trans(new TBufferedTransport(buffer, 3)); + proto.reset(new TDenseProtocol(buff_trans)); + checkout(0x0000000100000000ull, "\x90\x80\x80\x80\x00"); + checkout(0x0000000200000000ull, "\xA0\x80\x80\x80\x00"); + checkout(0x0000000300000000ull, "\xB0\x80\x80\x80\x00"); + checkout(0x0000000700000000ull, "\xF0\x80\x80\x80\x00"); + checkout(0x00000007F0000000ull, "\xFF\x80\x80\x80\x00"); + checkout(0x00000007FFFFFFFFull, "\xFF\xFF\xFF\xFF\x7F"); + checkout(0x0000000800000000ull, "\x81\x80\x80\x80\x80\x00"); + checkout(0x1FFFFFFFFFFFFFFFull, "\x9F\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F"); + checkout(0x7FFFFFFFFFFFFFFFull, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F"); + checkout(0xFFFFFFFFFFFFFFFFull, "\x81\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F"); + + // Test optional stuff. + proto.reset(new TDenseProtocol(buffer)); + proto->setTypeSpec(ManyOpt::local_reflection); + ManyOpt mo1, mo2, mo3, mo4, mo5, mo6; + mo1.opt1 = 923759347; + mo1.opt2 = 392749274; + mo1.opt3 = 395739402; + mo1.def4 = 294730928; + mo1.opt5 = 394309218; + mo1.opt6 = 832194723; + mo1.__isset.opt1 = true; + mo1.__isset.opt2 = true; + mo1.__isset.opt3 = true; + mo1.__isset.def4 = true; + mo1.__isset.opt5 = true; + mo1.__isset.opt6 = true; + + mo1.write(proto.get()); + mo2.read(proto.get()); + + assert(mo2.__isset.opt1 == true); + assert(mo2.__isset.opt2 == true); + assert(mo2.__isset.opt3 == true); + assert(mo2.__isset.def4 == true); + assert(mo2.__isset.opt5 == true); + assert(mo2.__isset.opt6 == true); + + assert(mo1 == mo2); + + mo1.__isset.opt1 = false; + mo1.__isset.opt3 = false; + mo1.__isset.opt5 = false; + + mo1.write(proto.get()); + mo3.read(proto.get()); + + assert(mo3.__isset.opt1 == false); + assert(mo3.__isset.opt2 == true); + assert(mo3.__isset.opt3 == false); + assert(mo3.__isset.def4 == true); + assert(mo3.__isset.opt5 == false); + assert(mo3.__isset.opt6 == true); + + assert(mo1 == mo3); + + mo1.__isset.opt1 = true; + mo1.__isset.opt3 = true; + mo1.__isset.opt5 = true; + mo1.__isset.opt2 = false; + mo1.__isset.opt6 = false; + + mo1.write(proto.get()); + mo4.read(proto.get()); + + assert(mo4.__isset.opt1 == true); + assert(mo4.__isset.opt2 == false); + assert(mo4.__isset.opt3 == true); + assert(mo4.__isset.def4 == true); + assert(mo4.__isset.opt5 == true); + assert(mo4.__isset.opt6 == false); + + assert(mo1 == mo4); + + mo1.__isset.opt1 = false; + mo1.__isset.opt5 = false; + + mo1.write(proto.get()); + mo5.read(proto.get()); + + assert(mo5.__isset.opt1 == false); + assert(mo5.__isset.opt2 == false); + assert(mo5.__isset.opt3 == true); + assert(mo5.__isset.def4 == true); + assert(mo5.__isset.opt5 == false); + assert(mo5.__isset.opt6 == false); + + assert(mo1 == mo5); + + mo1.__isset.opt3 = false; + + mo1.write(proto.get()); + mo6.read(proto.get()); + + assert(mo6.__isset.opt1 == false); + assert(mo6.__isset.opt2 == false); + assert(mo6.__isset.opt3 == false); + assert(mo6.__isset.def4 == true); + assert(mo6.__isset.opt5 == false); + assert(mo6.__isset.opt6 == false); + + assert(mo1 == mo6); + + + // Test fingerprint checking stuff. + + { + // Default and required have the same fingerprint. + Tricky1 t1; + Tricky3 t3; + assert(string(Tricky1::ascii_fingerprint) == Tricky3::ascii_fingerprint); + proto->setTypeSpec(Tricky1::local_reflection); + t1.im_default = 227; + t1.write(proto.get()); + proto->setTypeSpec(Tricky3::local_reflection); + t3.read(proto.get()); + assert(t3.im_required == 227); + } + + { + // Optional changes things. + Tricky1 t1; + Tricky2 t2; + assert(string(Tricky1::ascii_fingerprint) != Tricky2::ascii_fingerprint); + proto->setTypeSpec(Tricky1::local_reflection); + t1.im_default = 227; + t1.write(proto.get()); + try { + proto->setTypeSpec(Tricky2::local_reflection); + t2.read(proto.get()); + assert(false); + } catch (TProtocolException& ex) { + buffer->resetBuffer(); + } + } + + { + // Holy cow. We can use the Tricky1 typespec with the Tricky2 structure. + Tricky1 t1; + Tricky2 t2; + proto->setTypeSpec(Tricky1::local_reflection); + t1.im_default = 227; + t1.write(proto.get()); + t2.read(proto.get()); + assert(t2.__isset.im_optional == true); + assert(t2.im_optional == 227); + } + + { + // And totally off the wall. + Tricky1 t1; + OneOfEach ooe2; + assert(string(Tricky1::ascii_fingerprint) != OneOfEach::ascii_fingerprint); + proto->setTypeSpec(Tricky1::local_reflection); + t1.im_default = 227; + t1.write(proto.get()); + try { + proto->setTypeSpec(OneOfEach::local_reflection); + ooe2.read(proto.get()); + assert(false); + } catch (TProtocolException& ex) { + buffer->resetBuffer(); + } + } + + // Okay, this is really off the wall. + // Just don't crash. + cout << "Starting fuzz test. This takes a while. (20 dots.)" << endl; + std::srand(12345); + for (int i = 0; i < 2000; i++) { + if (i % 100 == 0) { + cout << "."; + cout.flush(); + } + buffer->resetBuffer(); + // Make sure the fingerprint prefix is right. + buffer->write(Nesting::binary_fingerprint, 4); + for (int j = 0; j < 1024*1024; j++) { + uint8_t r = std::rand(); + buffer->write(&r, 1); + } + Nesting n; + proto->setTypeSpec(OneOfEach::local_reflection); + try { + n.read(proto.get()); + } catch (TProtocolException& ex) { + } catch (TTransportException& ex) { + } + } + cout << endl; + + return 0; +} diff --git a/test/DocTest.thrift b/test/DocTest.thrift new file mode 100644 index 00000000..cb355ae3 --- /dev/null +++ b/test/DocTest.thrift @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Program doctext. + * + * Seriously, this is the documentation for this whole program. + */ + +namespace java thrift.test +namespace cpp thrift.test + +// C++ comment +/* c style comment */ + +# the new unix comment + +/** Some doc text goes here. Wow I am [nesting these] (no more nesting.) */ +enum Numberz +{ + + /** This is how to document a parameter */ + ONE = 1, + + /** And this is a doc for a parameter that has no specific value assigned */ + TWO, + + THREE, + FIVE = 5, + SIX, + EIGHT = 8 +} + +/** This is how you would do a typedef doc */ +typedef i64 UserId + +/** And this is where you would document a struct */ +struct Xtruct +{ + + /** And the members of a struct */ + 1: string string_thing + + /** doct text goes before a comma */ + 4: byte byte_thing, + + 9: i32 i32_thing, + 11: i64 i64_thing +} + +/** + * You can document constants now too. Yeehaw! + */ +const i32 INT32CONSTANT = 9853 +const i16 INT16CONSTANT = 1616 +/** Everyone get in on the docu-action! */ +const map MAPCONSTANT = {'hello':'world', 'goodnight':'moon'} + +struct Xtruct2 +{ + 1: byte byte_thing, + 2: Xtruct struct_thing, + 3: i32 i32_thing +} + +/** Struct insanity */ +struct Insanity +{ + + /** This is doc for field 1 */ + 1: map userMap, + + /** And this is doc for field 2 */ + 2: list xtructs +} + +exception Xception { + 1: i32 errorCode, + 2: string message +} + +exception Xception2 { + 1: i32 errorCode, + 2: Xtruct struct_thing +} + +/* C1 */ +/** Doc */ +/* C2 */ +/* C3 */ +struct EmptyStruct {} + +struct OneField { + 1: EmptyStruct field +} + +/** This is where you would document a Service */ +service ThriftTest +{ + + /** And this is how you would document functions in a service */ + void testVoid(), + string testString(1: string thing), + byte testByte(1: byte thing), + i32 testI32(1: i32 thing), + + /** Like this one */ + i64 testI64(1: i64 thing), + double testDouble(1: double thing), + Xtruct testStruct(1: Xtruct thing), + Xtruct2 testNest(1: Xtruct2 thing), + map testMap(1: map thing), + set testSet(1: set thing), + list testList(1: list thing), + + /** This is an example of a function with params documented */ + Numberz testEnum( + + /** This param is a thing */ + 1: Numberz thing + + ), + + UserId testTypedef(1: UserId thing), + + map> testMapMap(1: i32 hello), + + /* So you think you've got this all worked, out eh? */ + map> testInsanity(1: Insanity argument), + +} + +/// This style of Doxy-comment doesn't work. +typedef i32 SorryNoGo + +/** + * This is a trivial example of a multiline docstring. + */ +typedef i32 TrivialMultiLine + +/** + * This is the cannonical example + * of a multiline docstring. + */ +typedef i32 StandardMultiLine + +/** + * The last line is non-blank. + * I said non-blank! */ +typedef i32 LastLine + +/** Both the first line + * are non blank. ;-) + * and the last line */ +typedef i32 FirstAndLastLine + +/** + * INDENTED TITLE + * The text is less indented. + */ +typedef i32 IndentedTitle + +/** First line indented. + * Unfortunately, this does not get indented. + */ +typedef i32 FirstLineIndent + + +/** + * void code_in_comment() { + * printf("hooray code!"); + * } + */ +typedef i32 CodeInComment + + /** + * Indented Docstring. + * This whole docstring is indented. + * This line is indented further. + */ +typedef i32 IndentedDocstring + +/** Irregular docstring. + * We will have to punt + * on this thing */ +typedef i32 Irregular1 + +/** + * note the space + * before these lines +* but not this + * one + */ +typedef i32 Irregular2 + +/** +* Flush against +* the left. +*/ +typedef i32 Flush + +/** + No stars in this one. + It should still work fine, though. + Including indenting. + */ +typedef i32 NoStars + +/** Trailing whitespace +Sloppy trailing whitespace +is truncated. */ +typedef i32 TrailingWhitespace + +/** + * This is a big one. + * + * We'll have some blank lines in it. + * + * void as_well_as(some code) { + * puts("YEEHAW!"); + * } + */ +typedef i32 BigDog + +/** +* +* +*/ +typedef i32 TotallyDegenerate + +/* THE END */ diff --git a/test/FastbinaryTest.py b/test/FastbinaryTest.py new file mode 100755 index 00000000..7f6efae6 --- /dev/null +++ b/test/FastbinaryTest.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +r""" +thrift --gen py DebugProtoTest.thrift +./FastbinaryTest.py +""" + +# TODO(dreiss): Test error cases. Check for memory leaks. + +import sys +sys.path.append('./gen-py') + +import math +from DebugProtoTest import Srv +from DebugProtoTest.ttypes import * +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol + +import timeit +from cStringIO import StringIO +from copy import deepcopy +from pprint import pprint + +class TDevNullTransport(TTransport.TTransportBase): + def __init__(self): + pass + def isOpen(self): + return True + +ooe1 = OneOfEach() +ooe1.im_true = True; +ooe1.im_false = False; +ooe1.a_bite = 0xd6; +ooe1.integer16 = 27000; +ooe1.integer32 = 1<<24; +ooe1.integer64 = 6000 * 1000 * 1000; +ooe1.double_precision = math.pi; +ooe1.some_characters = "Debug THIS!"; +ooe1.zomg_unicode = "\xd7\n\a\t"; + +ooe2 = OneOfEach(); +ooe2.integer16 = 16; +ooe2.integer32 = 32; +ooe2.integer64 = 64; +ooe2.double_precision = (math.sqrt(5)+1)/2; +ooe2.some_characters = ":R (me going \"rrrr\")"; +ooe2.zomg_unicode = "\xd3\x80\xe2\x85\xae\xce\x9d\x20"\ + "\xd0\x9d\xce\xbf\xe2\x85\xbf\xd0\xbe"\ + "\xc9\xa1\xd0\xb3\xd0\xb0\xcf\x81\xe2\x84\x8e"\ + "\x20\xce\x91\x74\x74\xce\xb1\xe2\x85\xbd\xce\xba"\ + "\xc7\x83\xe2\x80\xbc"; + +hm = HolyMoley({"big":[], "contain":set(), "bonks":{}}) +hm.big.append(ooe1) +hm.big.append(ooe2) +hm.big[0].a_bite = 0x22; +hm.big[1].a_bite = 0x22; + +hm.contain.add(("and a one", "and a two")) +hm.contain.add(("then a one, two", "three!", "FOUR!")) +hm.contain.add(()) + +hm.bonks["nothing"] = []; +hm.bonks["something"] = [ + Bonk({"type":1, "message":"Wait."}), + Bonk({"type":2, "message":"What?"}), +] +hm.bonks["poe"] = [ + Bonk({"type":3, "message":"quoth"}), + Bonk({"type":4, "message":"the raven"}), + Bonk({"type":5, "message":"nevermore"}), +] + +rs = RandomStuff() +rs.a = 1 +rs.b = 2 +rs.c = 3 +rs.myintlist = range(20) +rs.maps = {1:Wrapper({"foo":Empty()}),2:Wrapper({"foo":Empty()})} +rs.bigint = 124523452435L +rs.triple = 3.14 + +# make sure this splits two buffers in a buffered protocol +rshuge = RandomStuff() +rshuge.myintlist=range(10000) + +my_zero = Srv.Janky_result({"arg":5}) + +def checkWrite(o): + trans_fast = TTransport.TMemoryBuffer() + trans_slow = TTransport.TMemoryBuffer() + prot_fast = TBinaryProtocol.TBinaryProtocolAccelerated(trans_fast) + prot_slow = TBinaryProtocol.TBinaryProtocol(trans_slow) + + o.write(prot_fast) + o.write(prot_slow) + ORIG = trans_slow.getvalue() + MINE = trans_fast.getvalue() + if ORIG != MINE: + print "mine: %s\norig: %s" % (repr(MINE), repr(ORIG)) + +def checkRead(o): + prot = TBinaryProtocol.TBinaryProtocol(TTransport.TMemoryBuffer()) + o.write(prot) + + slow_version_binary = prot.trans.getvalue() + + prot = TBinaryProtocol.TBinaryProtocolAccelerated( + TTransport.TMemoryBuffer(slow_version_binary)) + c = o.__class__() + c.read(prot) + if c != o: + print "copy: " + pprint(eval(repr(c))) + print "orig: " + pprint(eval(repr(o))) + + prot = TBinaryProtocol.TBinaryProtocolAccelerated( + TTransport.TBufferedTransport( + TTransport.TMemoryBuffer(slow_version_binary))) + c = o.__class__() + c.read(prot) + if c != o: + print "copy: " + pprint(eval(repr(c))) + print "orig: " + pprint(eval(repr(o))) + + +def doTest(): + checkWrite(hm) + no_set = deepcopy(hm) + no_set.contain = set() + checkRead(no_set) + checkWrite(rs) + checkRead(rs) + checkWrite(rshuge) + checkRead(rshuge) + checkWrite(my_zero) + checkRead(my_zero) + checkRead(Backwards({"first_tag2":4, "second_tag1":2})) + + # One case where the serialized form changes, but only superficially. + o = Backwards({"first_tag2":4, "second_tag1":2}) + trans_fast = TTransport.TMemoryBuffer() + trans_slow = TTransport.TMemoryBuffer() + prot_fast = TBinaryProtocol.TBinaryProtocolAccelerated(trans_fast) + prot_slow = TBinaryProtocol.TBinaryProtocol(trans_slow) + + o.write(prot_fast) + o.write(prot_slow) + ORIG = trans_slow.getvalue() + MINE = trans_fast.getvalue() + if ORIG == MINE: + print "That shouldn't happen." + + + prot = TBinaryProtocol.TBinaryProtocolAccelerated(TTransport.TMemoryBuffer()) + o.write(prot) + prot = TBinaryProtocol.TBinaryProtocol( + TTransport.TMemoryBuffer( + prot.trans.getvalue())) + c = o.__class__() + c.read(prot) + if c != o: + print "copy: " + pprint(eval(repr(c))) + print "orig: " + pprint(eval(repr(o))) + + + +def doBenchmark(): + + iters = 25000 + + setup = """ +from __main__ import hm, rs, TDevNullTransport +from thrift.protocol import TBinaryProtocol +trans = TDevNullTransport() +prot = TBinaryProtocol.TBinaryProtocol%s(trans) +""" + + setup_fast = setup % "Accelerated" + setup_slow = setup % "" + + print "Starting Benchmarks" + + print "HolyMoley Standard = %f" % \ + timeit.Timer('hm.write(prot)', setup_slow).timeit(number=iters) + print "HolyMoley Acceler. = %f" % \ + timeit.Timer('hm.write(prot)', setup_fast).timeit(number=iters) + + print "FastStruct Standard = %f" % \ + timeit.Timer('rs.write(prot)', setup_slow).timeit(number=iters) + print "FastStruct Acceler. = %f" % \ + timeit.Timer('rs.write(prot)', setup_fast).timeit(number=iters) + + + +doTest() +doBenchmark() + diff --git a/test/GenericHelpers.h b/test/GenericHelpers.h new file mode 100644 index 00000000..d661d8ba --- /dev/null +++ b/test/GenericHelpers.h @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef _THRIFT_TEST_GENERICHELPERS_H_ +#define _THRIFT_TEST_GENERICHELPERS_H_ 1 + +#include +#include +#include + +using boost::shared_ptr; +using namespace apache::thrift::protocol; + +/* ClassName Helper for cleaner exceptions */ +class ClassNames { + public: + template + static const char* getName() { return "Unknown type"; } +}; + +template <> const char* ClassNames::getName() { return "byte"; } +template <> const char* ClassNames::getName() { return "short"; } +template <> const char* ClassNames::getName() { return "int"; } +template <> const char* ClassNames::getName() { return "long"; } +template <> const char* ClassNames::getName() { return "double"; } +template <> const char* ClassNames::getName() { return "string"; } + +/* Generic Protocol I/O function for tests */ +class GenericIO { + public: + + /* Write functions */ + + static uint32_t write(shared_ptr proto, const int8_t& val) { + return proto->writeByte(val); + } + + static uint32_t write(shared_ptr proto, const int16_t& val) { + return proto->writeI16(val); + } + + static uint32_t write(shared_ptr proto, const int32_t& val) { + return proto->writeI32(val); + } + + static uint32_t write(shared_ptr proto, const double& val) { + return proto->writeDouble(val); + } + + static uint32_t write(shared_ptr proto, const int64_t& val) { + return proto->writeI64(val); + } + + static uint32_t write(shared_ptr proto, const std::string& val) { + return proto->writeString(val); + } + + /* Read functions */ + + static uint32_t read(shared_ptr proto, int8_t& val) { + return proto->readByte(val); + } + + static uint32_t read(shared_ptr proto, int16_t& val) { + return proto->readI16(val); + } + + static uint32_t read(shared_ptr proto, int32_t& val) { + return proto->readI32(val); + } + + static uint32_t read(shared_ptr proto, int64_t& val) { + return proto->readI64(val); + } + + static uint32_t read(shared_ptr proto, double& val) { + return proto->readDouble(val); + } + + static uint32_t read(shared_ptr proto, std::string& val) { + return proto->readString(val); + } + +}; + +#endif diff --git a/test/JSONProtoTest.cpp b/test/JSONProtoTest.cpp new file mode 100644 index 00000000..66813569 --- /dev/null +++ b/test/JSONProtoTest.cpp @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include "gen-cpp/DebugProtoTest_types.h" + +int main() { + using std::cout; + using std::endl; + using namespace thrift::test::debug; + using apache::thrift::transport::TMemoryBuffer; + using apache::thrift::protocol::TJSONProtocol; + + OneOfEach ooe; + ooe.im_true = true; + ooe.im_false = false; + ooe.a_bite = 0xd6; + ooe.integer16 = 27000; + ooe.integer32 = 1<<24; + ooe.integer64 = (uint64_t)6000 * 1000 * 1000; + ooe.double_precision = M_PI; + ooe.some_characters = "JSON THIS! \"\1"; + ooe.zomg_unicode = "\xd7\n\a\t"; + ooe.base64 = "\1\2\3\255"; + cout << apache::thrift::ThriftJSONString(ooe) << endl << endl; + + + Nesting n; + n.my_ooe = ooe; + n.my_ooe.integer16 = 16; + n.my_ooe.integer32 = 32; + n.my_ooe.integer64 = 64; + n.my_ooe.double_precision = (std::sqrt(5)+1)/2; + n.my_ooe.some_characters = ":R (me going \"rrrr\")"; + n.my_ooe.zomg_unicode = "\xd3\x80\xe2\x85\xae\xce\x9d\x20" + "\xd0\x9d\xce\xbf\xe2\x85\xbf\xd0\xbe\xc9\xa1\xd0\xb3\xd0\xb0\xcf\x81\xe2\x84\x8e" + "\x20\xce\x91\x74\x74\xce\xb1\xe2\x85\xbd\xce\xba\xc7\x83\xe2\x80\xbc"; + n.my_bonk.type = 31337; + n.my_bonk.message = "I am a bonk... xor!"; + + cout << apache::thrift::ThriftJSONString(n) << endl << endl; + + + HolyMoley hm; + + hm.big.push_back(ooe); + hm.big.push_back(n.my_ooe); + hm.big[0].a_bite = 0x22; + hm.big[1].a_bite = 0x33; + + std::vector stage1; + stage1.push_back("and a one"); + stage1.push_back("and a two"); + hm.contain.insert(stage1); + stage1.clear(); + stage1.push_back("then a one, two"); + stage1.push_back("three!"); + stage1.push_back("FOUR!!"); + hm.contain.insert(stage1); + stage1.clear(); + hm.contain.insert(stage1); + + std::vector stage2; + hm.bonks["nothing"] = stage2; + stage2.resize(stage2.size()+1); + stage2.back().type = 1; + stage2.back().message = "Wait."; + stage2.resize(stage2.size()+1); + stage2.back().type = 2; + stage2.back().message = "What?"; + hm.bonks["something"] = stage2; + stage2.clear(); + stage2.resize(stage2.size()+1); + stage2.back().type = 3; + stage2.back().message = "quoth"; + stage2.resize(stage2.size()+1); + stage2.back().type = 4; + stage2.back().message = "the raven"; + stage2.resize(stage2.size()+1); + stage2.back().type = 5; + stage2.back().message = "nevermore"; + hm.bonks["poe"] = stage2; + + cout << apache::thrift::ThriftJSONString(hm) << endl << endl; + + boost::shared_ptr buffer(new TMemoryBuffer()); + boost::shared_ptr proto(new TJSONProtocol(buffer)); + + + cout << "Testing ooe" << endl; + + ooe.write(proto.get()); + OneOfEach ooe2; + ooe2.read(proto.get()); + + assert(ooe == ooe2); + + + cout << "Testing hm" << endl; + + hm.write(proto.get()); + HolyMoley hm2; + hm2.read(proto.get()); + + assert(hm == hm2); + + hm2.big[0].a_bite = 0xFF; + + assert(hm != hm2); + + Doubles dub; + dub.nan = HUGE_VAL/HUGE_VAL; + dub.inf = HUGE_VAL; + dub.neginf = -HUGE_VAL; + dub.repeating = 10.0/3.0; + dub.big = 1E+305; + dub.small = 1E-305; + dub.zero = 0.0; + dub.negzero = -0.0; + cout << apache::thrift::ThriftJSONString(dub) << endl << endl; + + cout << "Testing base" << endl; + + Base64 base; + base.a = 123; + base.b1 = "1"; + base.b2 = "12"; + base.b3 = "123"; + base.b4 = "1234"; + base.b5 = "12345"; + base.b6 = "123456"; + + base.write(proto.get()); + Base64 base2; + base2.read(proto.get()); + + assert(base == base2); + + return 0; +} diff --git a/test/JavaBeansTest.thrift b/test/JavaBeansTest.thrift new file mode 100644 index 00000000..02bf98d6 --- /dev/null +++ b/test/JavaBeansTest.thrift @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace java thrift.test + +struct OneOfEachBeans { + 1: bool boolean_field, + 2: byte a_bite, + 3: i16 integer16, + 4: i32 integer32, + 5: i64 integer64, + 6: double double_precision, + 7: string some_characters, + 8: binary base64, + 9: list byte_list, + 10: list i16_list, + 11: list i64_list +} diff --git a/test/Makefile.am b/test/Makefile.am new file mode 100644 index 00000000..1226935d --- /dev/null +++ b/test/Makefile.am @@ -0,0 +1,178 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +SUBDIRS = + +if WITH_PYTHON +SUBDIRS += py +endif + +if WITH_RUBY +SUBDIRS += rb +endif + +noinst_LTLIBRARIES = libtestgencpp.la +libtestgencpp_la_SOURCES = \ + gen-cpp/DebugProtoTest_types.cpp \ + gen-cpp/OptionalRequiredTest_types.cpp \ + gen-cpp/DebugProtoTest_types.cpp \ + gen-cpp/ThriftTest_types.cpp \ + gen-cpp/DebugProtoTest_types.h \ + gen-cpp/OptionalRequiredTest_types.h \ + gen-cpp/ThriftTest_types.h \ + ThriftTest_extras.cpp \ + DebugProtoTest_extras.cpp + +ThriftTest_extras.o: gen-cpp/ThriftTest_types.h +DebugProtoTest_extras.o: gen-cpp/DebugProtoTest_types.h + +libtestgencpp_la_LIBADD = $(top_builddir)/lib/cpp/libthrift.la + +noinst_PROGRAMS = Benchmark + +Benchmark_SOURCES = \ + Benchmark.cpp + +Benchmark_LDADD = libtestgencpp.la + +check_PROGRAMS = \ + TFDTransportTest \ + TPipedTransportTest \ + DebugProtoTest \ + JSONProtoTest \ + OptionalRequiredTest \ + AllProtocolsTest \ + UnitTests + +TESTS = \ + $(check_PROGRAMS) + +UnitTests_SOURCES = \ + UnitTestMain.cpp \ + TMemoryBufferTest.cpp \ + TBufferBaseTest.cpp + +UnitTests_LDADD = libtestgencpp.la + +# +# TFDTransportTest +# +TFDTransportTest_SOURCES = \ + TFDTransportTest.cpp + +TFDTransportTest_LDADD = \ + $(top_builddir)/lib/cpp/libthrift.la + + +# +# TPipedTransportTest +# +TPipedTransportTest_SOURCES = \ + TPipedTransportTest.cpp + +TPipedTransportTest_LDADD = \ + $(top_builddir)/lib/cpp/libthrift.la + +# +# AllProtocolsTest +# +AllProtocolsTest_SOURCES = \ + AllProtocolTests.cpp \ + AllProtocolTests.tcc \ + GenericHelpers.h + +AllProtocolsTest_LDADD = libtestgencpp.la + +# +# DebugProtoTest +# +DebugProtoTest_SOURCES = \ + DebugProtoTest.cpp + +DebugProtoTest_LDADD = libtestgencpp.la + + +# +# JSONProtoTest +# +JSONProtoTest_SOURCES = \ + JSONProtoTest.cpp + +JSONProtoTest_LDADD = libtestgencpp.la + +# +# OptionalRequiredTest +# +OptionalRequiredTest_SOURCES = \ + OptionalRequiredTest.cpp + +OptionalRequiredTest_LDADD = libtestgencpp.la + + +# +# Common thrift code generation rules +# +THRIFT = $(top_builddir)/compiler/cpp/thrift + +gen-cpp/DebugProtoTest_types.cpp gen-cpp/DebugProtoTest_types.h: DebugProtoTest.thrift + $(THRIFT) --gen cpp:dense $< + +gen-cpp/OptionalRequiredTest_types.cpp gen-cpp/OptionalRequiredTest_types.h: OptionalRequiredTest.thrift + $(THRIFT) --gen cpp:dense $< + +gen-cpp/Service.cpp gen-cpp/StressTest_types.cpp: StressTest.thrift + $(THRIFT) --gen cpp:dense $< + +gen-cpp/SecondService.cpp gen-cpp/ThriftTest_constants.cpp gen-cpp/ThriftTest.cpp gen-cpp/ThriftTest_types.cpp gen-cpp/ThriftTest_types.h: ThriftTest.thrift + $(THRIFT) --gen cpp:dense $< + +INCLUDES = \ + -I$(top_srcdir)/lib/cpp/src + +AM_CPPFLAGS = $(BOOST_CPPFLAGS) + +clean-local: + $(RM) -r gen-cpp + +EXTRA_DIST = \ + cpp \ + threads \ + csharp \ + py \ + rb \ + perl \ + php \ + erl \ + hs \ + ocaml \ + AnnotationTest.thrift \ + BrokenConstants.thrift \ + ConstantsDemo.thrift \ + DebugProtoTest.thrift \ + DenseLinkingTest.thrift \ + DocTest.thrift \ + JavaBeansTest.thrift \ + ManyTypedefs.thrift \ + OptionalRequiredTest.thrift \ + SmallTest.thrift \ + StressTest.thrift \ + ThriftTest.thrift \ + ZlibTest.cpp \ + DenseProtoTest.cpp \ + FastbinaryTest.py diff --git a/test/ManyTypedefs.thrift b/test/ManyTypedefs.thrift new file mode 100644 index 00000000..d194b63c --- /dev/null +++ b/test/ManyTypedefs.thrift @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// This is to make sure you don't mess something up when you change typedef code. +// Generate it with the old and new thrift and make sure they are the same. +/* +rm -rf gen-* orig-* +mkdir old new +thrift --gen cpp --gen java --gen php --gen phpi --gen py --gen rb --gen xsd --gen perl --gen ocaml --gen erl --gen hs --strict ManyTypedefs.thrift +mv gen-* old +../compiler/cpp/thrift --gen cpp --gen java --gen php --gen phpi --gen py --gen rb --gen xsd --gen perl --gen ocaml --gen erl --gen hs --strict ManyTypedefs.thrift +mv gen-* new +diff -ur old new +rm -rf old new +# There should be no output. +*/ + +typedef i32 int32 +typedef list> biglist + +struct struct1 { + 1: int32 myint; + 2: biglist mylist; +} + +exception exception1 { + 1: biglist alist; + 2: struct1 mystruct; +} + +service AService { + struct1 method1(1: int32 myint) throws (1: exception1 exn); + biglist method2(); +} diff --git a/test/OptionalRequiredTest.cpp b/test/OptionalRequiredTest.cpp new file mode 100644 index 00000000..5743ce30 --- /dev/null +++ b/test/OptionalRequiredTest.cpp @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include "gen-cpp/OptionalRequiredTest_types.h" + +using std::cout; +using std::endl; +using std::map; +using std::string; +using namespace thrift::test; +using namespace apache::thrift; +using namespace apache::thrift::transport; +using namespace apache::thrift::protocol; + + +/* +template +void trywrite(const Struct& s, bool should_work) { + bool worked; + try { + TBinaryProtocol protocol(boost::shared_ptr(new TMemoryBuffer)); + s.write(&protocol); + worked = true; + } catch (TProtocolException & ex) { + worked = false; + } + assert(worked == should_work); +} +*/ + +template +void write_to_read(const Struct1 & w, Struct2 & r) { + TBinaryProtocol protocol(boost::shared_ptr(new TMemoryBuffer)); + w.write(&protocol); + r.read(&protocol); +} + + +int main() { + + cout << "This old school struct should have three fields." << endl; + { + OldSchool o; + cout << ThriftDebugString(o) << endl; + } + cout << endl; + + cout << "Setting a value before setting isset." << endl; + { + Simple s; + cout << ThriftDebugString(s) << endl; + s.im_optional = 10; + cout << ThriftDebugString(s) << endl; + s.__isset.im_optional = true; + cout << ThriftDebugString(s) << endl; + } + cout << endl; + + cout << "Setting isset before setting a value." << endl; + { + Simple s; + cout << ThriftDebugString(s) << endl; + s.__isset.im_optional = true; + cout << ThriftDebugString(s) << endl; + s.im_optional = 10; + cout << ThriftDebugString(s) << endl; + } + cout << endl; + + // Write-to-read with optional fields. + { + Simple s1, s2, s3; + s1.im_optional = 10; + assert(!s1.__isset.im_default); + //assert(!s1.__isset.im_required); // Compile error. + assert(!s1.__isset.im_optional); + + write_to_read(s1, s2); + + assert( s2.__isset.im_default); + //assert( s2.__isset.im_required); // Compile error. + assert(!s2.__isset.im_optional); + assert(s3.im_optional == 0); + + s1.__isset.im_optional = true; + write_to_read(s1, s3); + + assert( s3.__isset.im_default); + //assert( s3.__isset.im_required); // Compile error. + assert( s3.__isset.im_optional); + assert(s3.im_optional == 10); + } + + // Writing between optional and default. + { + Tricky1 t1; + Tricky2 t2; + + t2.im_optional = 10; + write_to_read(t2, t1); + write_to_read(t1, t2); + assert(!t1.__isset.im_default); + assert( t2.__isset.im_optional); + assert(t1.im_default == t2.im_optional); + assert(t1.im_default == 0); + } + + // Writing between default and required. + { + Tricky1 t1; + Tricky3 t3; + write_to_read(t1, t3); + write_to_read(t3, t1); + assert(t1.__isset.im_default); + } + + // Writing between optional and required. + { + Tricky2 t2; + Tricky3 t3; + t2.__isset.im_optional = true; + write_to_read(t2, t3); + write_to_read(t3, t2); + } + + // Mu-hu-ha-ha-ha! + { + Tricky2 t2; + Tricky3 t3; + try { + write_to_read(t2, t3); + abort(); + } + catch (TProtocolException& ex) {} + + write_to_read(t3, t2); + assert(t2.__isset.im_optional); + } + + cout << "Complex struct, simple test." << endl; + { + Complex c; + cout << ThriftDebugString(c) << endl; + } + + + { + Tricky1 t1; + Tricky2 t2; + // Compile error. + //(void)(t1 == t2); + } + + { + OldSchool o1, o2, o3; + assert(o1 == o2); + o1.im_int = o2.im_int = 10; + assert(o1 == o2); + o1.__isset.im_int = true; + o2.__isset.im_int = false; + assert(o1 == o2); + o1.im_int = 20; + o1.__isset.im_int = false; + assert(o1 != o2); + o1.im_int = 10; + assert(o1 == o2); + o1.im_str = o2.im_str = "foo"; + assert(o1 == o2); + o1.__isset.im_str = o2.__isset.im_str = true; + assert(o1 == o2); + map mymap; + mymap[1] = "bar"; + mymap[2] = "baz"; + o1.im_big.push_back(map()); + assert(o1 != o2); + o2.im_big.push_back(map()); + assert(o1 == o2); + o2.im_big.push_back(mymap); + assert(o1 != o2); + o1.im_big.push_back(mymap); + assert(o1 == o2); + + TBinaryProtocol protocol(boost::shared_ptr(new TMemoryBuffer)); + o1.write(&protocol); + + o1.im_big.push_back(mymap); + mymap[3] = "qux"; + o2.im_big.push_back(mymap); + assert(o1 != o2); + o1.im_big.back()[3] = "qux"; + assert(o1 == o2); + + o3.read(&protocol); + o3.im_big.push_back(mymap); + assert(o1 == o3); + + //cout << ThriftDebugString(o3) << endl; + } + + { + Tricky2 t1, t2; + assert(t1.__isset.im_optional == false); + assert(t2.__isset.im_optional == false); + assert(t1 == t2); + t1.im_optional = 5; + assert(t1 == t2); + t2.im_optional = 5; + assert(t1 == t2); + t1.__isset.im_optional = true; + assert(t1 != t2); + t2.__isset.im_optional = true; + assert(t1 == t2); + t1.im_optional = 10; + assert(t1 != t2); + t2.__isset.im_optional = false; + assert(t1 != t2); + } + + return 0; +} diff --git a/test/OptionalRequiredTest.thrift b/test/OptionalRequiredTest.thrift new file mode 100644 index 00000000..f7d1fd6a --- /dev/null +++ b/test/OptionalRequiredTest.thrift @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace cpp thrift.test +namespace java thrift.test + +struct OldSchool { + 1: i16 im_int; + 2: string im_str; + 3: list> im_big; +} + +struct Simple { + 1: /* :) */ i16 im_default; + 2: required i16 im_required; + 3: optional i16 im_optional; +} + +struct Tricky1 { + 1: /* :) */ i16 im_default; +} + +struct Tricky2 { + 1: optional i16 im_optional; +} + +struct Tricky3 { + 1: required i16 im_required; +} + +struct Complex { + 1: i16 cp_default; + 2: required i16 cp_required; + 3: optional i16 cp_optional; + 4: map the_map; + 5: required Simple req_simp; + 6: optional Simple opt_simp; +} + +struct ManyOpt { + 1: optional i32 opt1; + 2: optional i32 opt2; + 3: optional i32 opt3; + 4: i32 def4; + 5: optional i32 opt5; + 6: optional i32 opt6; +} + +struct JavaTestHelper { + 1: required i32 req_int; + 2: optional i32 opt_int; + 3: required string req_obj; + 4: optional string opt_obj; + 5: required binary req_bin; + 6: optional binary opt_bin; +} diff --git a/test/SmallTest.thrift b/test/SmallTest.thrift new file mode 100644 index 00000000..d0821c7f --- /dev/null +++ b/test/SmallTest.thrift @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +namespace rb TestNamespace + +struct Goodbyez { + 1: i32 val = 325; +} + +senum Thinger { + "ASDFKJ", + "r32)*F#@", + "ASDFLJASDF" +} + +struct BoolPasser { + 1: bool value = 1 +} + +struct Hello { + 1: i32 simple = 53, + 2: map complex = {23:532, 6243:632, 2355:532}, + 3: map> complexer, + 4: string words = "words", + 5: Goodbyez thinz = {'val' : 36632} +} + +const map> CMAP = { 235: {235:235}, 53:{53:53} } +const i32 CINT = 325; +const Hello WHOA = {'simple' : 532} + +exception Goodbye { + 1: i32 simple, + 2: map complex, + 3: map> complexer, +} + +service SmallService { + Thinger testThinger(1:Thinger bootz), + Hello testMe(1:i32 hello=64, 2: Hello wonk) throws (1: Goodbye g), + void testVoid() throws (1: Goodbye g), + i32 testI32(1:i32 boo) +} diff --git a/test/StressTest.thrift b/test/StressTest.thrift new file mode 100644 index 00000000..87c6e476 --- /dev/null +++ b/test/StressTest.thrift @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace cpp test.stress + +service Service { + + void echoVoid(), + byte echoByte(1: byte arg), + i32 echoI32(1: i32 arg), + i64 echoI64(1: i64 arg), + string echoString(1: string arg), + list echoList(1: list arg), + set echoSet(1: set arg), + map echoMap(1: map arg), +} + diff --git a/test/TBufferBaseTest.cpp b/test/TBufferBaseTest.cpp new file mode 100644 index 00000000..da3ce856 --- /dev/null +++ b/test/TBufferBaseTest.cpp @@ -0,0 +1,639 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +using std::string; +using boost::shared_ptr; +using apache::thrift::transport::TMemoryBuffer; +using apache::thrift::transport::TBufferedTransport; +using apache::thrift::transport::TFramedTransport; +using apache::thrift::transport::test::TShortReadTransport; + +#define foreach BOOST_FOREACH + +// Shamelessly copied from ZlibTransport. TODO: refactor. +unsigned int dist[][5000] = { + { 1<<15 }, + + { + 5,13,9,1,8,9,11,13,18,48,24,13,21,13,5,11,35,2,4,20,17,72,27,14,15,4,7,26, + 12,1,14,9,2,16,29,41,7,24,4,27,14,4,1,4,25,3,6,34,10,8,50,2,14,13,55,29,3, + 43,53,49,14,4,10,32,27,48,1,3,1,11,5,17,16,51,17,30,15,11,9,2,2,11,52,12,2, + 13,94,1,19,1,38,2,8,43,8,33,7,30,8,17,22,2,15,14,12,34,2,12,6,37,29,74,3, + 165,16,11,17,5,14,3,10,7,37,11,24,7,1,3,12,37,8,9,34,17,12,8,21,13,37,1,4, + 30,14,78,4,15,2,40,37,17,12,36,82,14,4,1,4,7,17,11,16,88,77,2,3,15,3,34,11, + 5,79,22,34,8,4,4,40,22,24,28,9,13,3,34,27,9,16,39,16,39,13,2,4,3,41,26,10,4, + 33,4,7,12,5,6,3,10,30,8,21,16,58,19,9,0,47,7,13,11,19,15,7,53,57,2,13,28,22, + 3,16,9,25,33,12,40,7,12,64,7,14,24,44,9,2,14,11,2,58,1,26,30,11,9,5,24,7,9, + 94,2,10,21,5,5,4,5,6,179,9,18,2,7,13,31,41,17,4,36,3,21,6,26,8,15,18,44,27, + 11,9,25,7,0,14,2,12,20,23,13,2,163,9,5,15,65,2,14,6,8,98,11,15,14,34,2,3,10, + 22,9,92,7,10,32,67,13,3,4,35,8,2,1,5,0,26,381,7,27,8,2,16,93,4,19,5,8,25,9, + 31,14,4,21,5,3,9,22,56,4,18,3,11,18,6,4,3,40,12,16,110,8,35,14,1,18,40,9,12, + 14,3,11,7,57,13,18,116,53,19,22,7,16,11,5,8,21,16,1,75,21,20,1,28,2,6,1,7, + 19,38,5,6,9,9,4,1,7,55,36,62,5,4,4,24,15,1,12,35,48,20,5,17,1,5,26,15,4,54, + 13,5,5,15,5,19,32,29,31,7,6,40,7,80,11,18,8,128,48,6,12,84,13,4,7,2,13,9,16, + 17,3,254,1,4,181,8,44,7,6,24,27,9,23,14,34,16,22,25,10,3,3,4,4,12,2,12,6,7, + 13,58,13,6,11,19,53,11,66,18,19,10,4,13,2,5,49,58,1,67,7,21,64,14,11,14,8,3, + 26,33,91,31,20,7,9,42,39,4,3,55,11,10,0,7,4,75,8,12,0,27,3,8,9,0,12,12,23, + 28,23,20,4,13,30,2,22,20,19,30,6,22,2,6,4,24,7,19,55,86,5,33,2,161,6,7,1,62, + 13,3,72,12,12,9,7,12,10,5,10,29,1,5,22,13,13,5,2,12,3,7,14,18,2,3,46,21,17, + 15,19,3,27,5,16,45,31,10,8,17,18,18,3,7,24,6,55,9,3,6,12,10,12,8,91,9,4,4,4, + 27,29,16,5,7,22,43,28,11,14,8,11,28,109,55,71,40,3,8,22,26,15,44,3,25,29,5, + 3,32,17,12,3,29,27,25,15,11,8,40,39,38,17,3,9,11,2,32,11,6,20,48,75,27,3,7, + 54,12,95,12,7,24,23,2,13,8,15,16,5,12,4,17,7,19,88,2,6,13,115,45,12,21,2,86, + 74,9,7,5,16,32,16,2,21,18,6,34,5,18,260,7,12,16,44,19,92,31,7,8,2,9,0,0,15, + 8,38,4,8,20,18,2,83,3,3,4,9,5,3,10,3,5,29,15,7,11,8,48,17,23,2,17,4,11,22, + 21,64,8,8,4,19,95,0,17,28,9,11,20,71,5,11,18,12,13,45,49,4,1,33,32,23,13,5, + 52,2,2,16,3,4,7,12,2,1,12,6,24,1,22,155,21,3,45,4,12,44,26,5,40,36,9,9,8,20, + 35,31,3,2,32,50,10,8,37,2,75,35,22,15,192,8,11,23,1,4,29,6,8,8,5,12,18,32,4, + 7,12,2,0,0,9,5,48,11,35,3,1,123,6,29,8,11,8,23,51,16,6,63,12,2,5,4,14,2,15, + 7,14,3,2,7,17,32,8,8,10,1,23,62,2,49,6,49,47,23,3,20,7,11,39,10,24,6,15,5,5, + 11,8,16,36,8,13,20,3,10,44,7,52,7,10,36,6,15,10,5,11,4,14,19,17,10,12,3,6, + 23,4,13,94,70,7,36,7,38,7,28,8,4,15,3,19,4,33,39,21,109,4,80,6,40,4,432,4,4, + 7,8,3,31,8,28,37,34,10,2,21,5,22,0,7,36,14,12,6,24,1,21,5,9,2,29,20,54,113, + 13,31,39,27,6,0,27,4,5,2,43,7,8,57,8,62,7,9,12,22,90,30,6,19,7,10,20,6,5,58, + 32,30,41,4,10,25,13,3,8,7,10,2,9,6,151,44,16,12,16,20,8,3,18,11,17,4,10,45, + 15,8,56,38,52,25,40,14,4,17,15,8,2,19,7,8,26,30,2,3,180,8,26,17,38,35,5,16, + 28,5,15,56,13,14,18,9,15,83,27,3,9,4,11,8,27,27,44,10,12,8,3,48,14,7,9,4,4, + 8,4,5,9,122,8,14,12,19,17,21,4,29,63,21,17,10,12,18,47,10,10,53,4,18,16,4,8, + 118,9,5,12,9,11,9,3,12,32,3,23,2,15,3,3,30,3,17,235,15,22,9,299,14,17,1,5, + 16,8,3,7,3,13,2,7,6,4,8,66,2,13,6,15,16,47,3,36,5,7,10,24,1,9,9,8,13,16,26, + 12,7,24,21,18,49,23,39,10,41,4,13,4,27,11,12,12,19,4,147,8,10,9,40,21,2,83, + 10,5,6,11,25,9,50,57,40,12,12,21,1,3,24,23,9,3,9,13,2,3,12,57,8,11,13,15,26, + 15,10,47,36,4,25,1,5,8,5,4,0,12,49,5,19,4,6,16,14,6,10,69,10,33,29,7,8,61, + 12,4,0,3,7,6,3,16,29,27,38,4,21,0,24,3,2,1,19,16,22,2,8,138,11,7,7,3,12,22, + 3,16,5,7,3,53,9,10,32,14,5,7,3,6,22,9,59,26,8,7,58,5,16,11,55,7,4,11,146,91, + 8,13,18,14,6,8,8,31,26,22,6,11,30,11,30,15,18,31,3,48,17,7,6,4,9,2,25,3,35, + 13,13,7,8,4,31,10,8,10,4,3,45,10,23,2,7,259,17,21,13,14,3,26,3,8,27,4,18,9, + 66,7,12,5,8,17,4,23,55,41,51,2,32,26,66,4,21,14,12,65,16,22,17,5,14,2,29,24, + 7,3,36,2,43,53,86,5,28,4,58,13,49,121,6,2,73,2,1,47,4,2,27,10,35,28,27,10, + 17,10,56,7,10,14,28,20,24,40,7,4,7,3,10,11,32,6,6,3,15,11,54,573,2,3,6,2,3, + 14,64,4,16,12,16,42,10,26,4,6,11,69,18,27,2,2,17,22,9,13,22,11,6,1,15,49,3, + 14,1 + }, + + { + 11,11,11,15,47,1,3,1,23,5,8,18,3,23,15,21,1,7,19,10,26,1,17,11,31,21,41,18, + 34,4,9,58,19,3,3,36,5,18,13,3,14,4,9,10,4,19,56,15,3,5,3,11,27,9,4,10,13,4, + 11,6,9,2,18,3,10,19,11,4,53,4,2,2,3,4,58,16,3,0,5,30,2,11,93,10,2,14,10,6,2, + 115,2,25,16,22,38,101,4,18,13,2,145,51,45,15,14,15,13,20,7,24,5,13,14,30,40, + 10,4,107,12,24,14,39,12,6,13,20,7,7,11,5,18,18,45,22,6,39,3,2,1,51,9,11,4, + 13,9,38,44,8,11,9,15,19,9,23,17,17,17,13,9,9,1,10,4,18,6,2,9,5,27,32,72,8, + 37,9,4,10,30,17,20,15,17,66,10,4,73,35,37,6,4,16,117,45,13,4,75,5,24,65,10, + 4,9,4,13,46,5,26,29,10,4,4,52,3,13,18,63,6,14,9,24,277,9,88,2,48,27,123,14, + 61,7,5,10,8,7,90,3,10,3,3,48,17,13,10,18,33,2,19,36,6,21,1,16,12,5,6,2,16, + 15,29,88,28,2,15,6,11,4,6,11,3,3,4,18,9,53,5,4,3,33,8,9,8,6,7,36,9,62,14,2, + 1,10,1,16,7,32,7,23,20,11,10,23,2,1,0,9,16,40,2,81,5,22,8,5,4,37,51,37,10, + 19,57,11,2,92,31,6,39,10,13,16,8,20,6,9,3,10,18,25,23,12,30,6,2,26,7,64,18, + 6,30,12,13,27,7,10,5,3,33,24,99,4,23,4,1,27,7,27,49,8,20,16,3,4,13,9,22,67, + 28,3,10,16,3,2,10,4,8,1,8,19,3,85,6,21,1,9,16,2,30,10,33,12,4,9,3,1,60,38,6, + 24,32,3,14,3,40,8,34,115,5,9,27,5,96,3,40,6,15,5,8,22,112,5,5,25,17,58,2,7, + 36,21,52,1,3,95,12,21,4,11,8,59,24,5,21,4,9,15,8,7,21,3,26,5,11,6,7,17,65, + 14,11,10,2,17,5,12,22,4,4,2,21,8,112,3,34,63,35,2,25,1,2,15,65,23,0,3,5,15, + 26,27,9,5,48,11,15,4,9,5,33,20,15,1,18,19,11,24,40,10,21,74,6,6,32,30,40,5, + 4,7,44,10,25,46,16,12,5,40,7,18,5,18,9,12,8,4,25,5,6,36,4,43,8,9,12,35,17,4, + 8,9,11,27,5,10,17,40,8,12,4,18,9,18,12,20,25,39,42,1,24,13,22,15,7,112,35,3, + 7,17,33,2,5,5,19,8,4,12,24,14,13,2,1,13,6,5,19,11,7,57,0,19,6,117,48,14,8, + 10,51,17,12,14,2,5,8,9,15,4,48,53,13,22,4,25,12,11,19,45,5,2,6,54,22,9,15,9, + 13,2,7,11,29,82,16,46,4,26,14,26,40,22,4,26,6,18,13,4,4,20,3,3,7,12,17,8,9, + 23,6,20,7,25,23,19,5,15,6,23,15,11,19,11,3,17,59,8,18,41,4,54,23,44,75,13, + 20,6,11,2,3,1,13,10,3,7,12,3,4,7,8,30,6,6,7,3,32,9,5,28,6,114,42,13,36,27, + 59,6,93,13,74,8,69,140,3,1,17,48,105,6,11,5,15,1,10,10,14,8,53,0,8,24,60,2, + 6,35,2,12,32,47,16,17,75,2,5,4,37,28,10,5,9,57,4,59,5,12,13,7,90,5,11,5,24, + 22,13,30,1,2,10,9,6,19,3,18,47,2,5,7,9,35,15,3,6,1,21,14,14,18,14,9,12,8,73, + 6,19,3,32,9,14,17,17,5,55,23,6,16,28,3,11,48,4,6,6,6,12,16,30,10,30,27,51, + 18,29,2,3,15,1,76,0,16,33,4,27,3,62,4,10,2,4,8,15,9,41,26,22,2,4,20,4,49,0, + 8,1,57,13,12,39,3,63,10,19,34,35,2,7,8,29,72,4,10,0,77,8,6,7,9,15,21,9,4,1, + 20,23,1,9,18,9,15,36,4,7,6,15,5,7,7,40,2,9,22,2,3,20,4,12,34,13,6,18,15,1, + 38,20,12,7,16,3,19,85,12,16,18,16,2,17,1,13,8,6,12,15,97,17,12,9,3,21,15,12, + 23,44,81,26,30,2,5,17,6,6,0,22,42,19,6,19,41,14,36,7,3,56,7,9,3,2,6,9,69,3, + 15,4,30,28,29,7,9,15,17,17,6,1,6,153,9,33,5,12,14,16,28,3,8,7,14,12,4,6,36, + 9,24,13,13,4,2,9,15,19,9,53,7,13,4,150,17,9,2,6,12,7,3,5,58,19,58,28,8,14,3, + 20,3,0,32,56,7,5,4,27,1,68,4,29,13,5,58,2,9,65,41,27,16,15,12,14,2,10,9,24, + 3,2,9,2,2,3,14,32,10,22,3,13,11,4,6,39,17,0,10,5,5,10,35,16,19,14,1,8,63,19, + 14,8,56,10,2,12,6,12,6,7,16,2,9,9,12,20,73,25,13,21,17,24,5,32,8,12,25,8,14, + 16,5,23,3,7,6,3,11,24,6,30,4,21,13,28,4,6,29,15,5,17,6,26,8,15,8,3,7,7,50, + 11,30,6,2,28,56,16,24,25,23,24,89,31,31,12,7,22,4,10,17,3,3,8,11,13,5,3,27, + 1,12,1,14,8,10,29,2,5,2,2,20,10,0,31,10,21,1,48,3,5,43,4,5,18,13,5,18,25,34, + 18,3,5,22,16,3,4,20,3,9,3,25,6,6,44,21,3,12,7,5,42,3,2,14,4,36,5,3,45,51,15, + 9,11,28,9,7,6,6,12,26,5,14,10,11,42,55,13,21,4,28,6,7,23,27,11,1,41,36,0,32, + 15,26,2,3,23,32,11,2,15,7,29,26,144,33,20,12,7,21,10,7,11,65,46,10,13,20,32, + 4,4,5,19,2,19,15,49,41,1,75,10,11,25,1,2,45,11,8,27,18,10,60,28,29,12,30,19, + 16,4,24,11,19,27,17,49,18,7,40,13,19,22,8,55,12,11,3,6,5,11,8,10,22,5,9,9, + 25,7,17,7,64,1,24,2,12,17,44,4,12,27,21,11,10,7,47,5,9,13,12,38,27,21,7,29, + 7,1,17,3,3,5,48,62,10,3,11,17,15,15,6,3,8,10,8,18,19,13,3,9,7,6,44,9,10,4, + 43,8,6,6,14,20,38,24,2,4,5,5,7,5,9,39,8,44,40,9,19,7,3,15,25,2,37,18,15,9,5, + 8,32,10,5,18,4,7,46,20,17,23,4,11,16,18,31,11,3,11,1,14,1,25,4,27,13,13,39, + 14,6,6,35,6,16,13,11,122,21,15,20,24,10,5,152,15,39,5,20,16,9,14,7,53,6,3,8, + 19,63,32,6,2,3,20,1,19,5,13,42,15,4,6,68,31,46,11,38,10,24,5,5,8,9,12,3,35, + 46,26,16,2,8,4,74,16,44,4,5,1,16,4,14,23,16,69,15,42,31,14,7,7,6,97,14,40,1, + 8,7,34,9,39,19,13,15,10,21,18,10,5,15,38,7,5,12,7,20,15,4,11,6,14,5,17,7,39, + 35,36,18,20,26,22,4,2,36,21,64,0,5,9,10,6,4,1,7,3,1,3,3,4,10,20,90,2,22,48, + 16,23,2,33,40,1,21,21,17,20,8,8,12,4,83,14,48,4,21,3,9,27,5,11,40,15,9,3,16, + 17,9,11,4,24,31,17,3,4,2,11,1,8,4,8,6,41,17,4,13,3,7,17,8,27,5,13,6,10,7,13, + 12,18,13,60,18,3,8,1,12,125,2,7,16,2,11,2,4,7,26,5,9,14,14,16,8,14,7,14,6,9, + 13,9,6,4,26,35,49,36,55,3,9,6,40,26,23,31,19,41,2,10,31,6,54,5,69,16,7,8,16, + 1,5,7,4,22,7,7,5,4,48,11,13,3,98,4,11,19,4,2,14,7,34,7,10,3,2,12,7,6,2,5,118 + }, +}; + +uint8_t data[1<<15]; +string data_str; +void init_data() { + static bool initted = false; + if (initted) return; + initted = true; + + // Repeatability. Kind of. + std::srand(42); + for (int i = 0; i < (int)(sizeof(data)/sizeof(data[0])); ++i) { + data[i] = (uint8_t)rand(); + } + + data_str.assign((char*)data, sizeof(data)); +} + + +BOOST_AUTO_TEST_SUITE( TBufferBaseTest ) + +BOOST_AUTO_TEST_CASE( test_MemoryBuffer_Write_GetBuffer ) { + init_data(); + + for (int d1 = 0; d1 < 3; d1++) { + TMemoryBuffer buffer(16); + int offset = 0; + int index = 0; + + while (offset < 1<<15) { + buffer.write(&data[offset], dist[d1][index]); + offset += dist[d1][index]; + index++; + } + + string output = buffer.getBufferAsString(); + BOOST_CHECK_EQUAL(data_str, output); + } +} + +BOOST_AUTO_TEST_CASE( test_MemoryBuffer_Write_Read ) { + init_data(); + + for (int d1 = 0; d1 < 3; d1++) { + for (int d2 = 0; d2 < 3; d2++) { + TMemoryBuffer buffer(16); + uint8_t data_out[1<<15]; + int offset; + int index; + + offset = 0; + index = 0; + while (offset < 1<<15) { + buffer.write(&data[offset], dist[d1][index]); + offset += dist[d1][index]; + index++; + } + + offset = 0; + index = 0; + while (offset < 1<<15) { + unsigned int got = buffer.read(&data_out[offset], dist[d2][index]); + BOOST_CHECK_EQUAL(got, dist[d2][index]); + offset += dist[d2][index]; + index++; + } + + BOOST_CHECK(!memcmp(data, data_out, sizeof(data))); + } + } +} + +BOOST_AUTO_TEST_CASE( test_MemoryBuffer_Write_ReadString ) { + init_data(); + + for (int d1 = 0; d1 < 3; d1++) { + for (int d2 = 0; d2 < 3; d2++) { + TMemoryBuffer buffer(16); + string output; + int offset; + int index; + + offset = 0; + index = 0; + while (offset < 1<<15) { + buffer.write(&data[offset], dist[d1][index]); + offset += dist[d1][index]; + index++; + } + + offset = 0; + index = 0; + while (offset < 1<<15) { + unsigned int got = buffer.readAppendToString(output, dist[d2][index]); + BOOST_CHECK_EQUAL(got, dist[d2][index]); + offset += dist[d2][index]; + index++; + } + + BOOST_CHECK_EQUAL(output, data_str); + } + } +} + +BOOST_AUTO_TEST_CASE( test_MemoryBuffer_Write_Read_Multi1 ) { + init_data(); + + // Do shorter writes and reads so we don't align to power-of-two boundaries. + + for (int d1 = 0; d1 < 3; d1++) { + for (int d2 = 0; d2 < 3; d2++) { + TMemoryBuffer buffer(16); + uint8_t data_out[1<<15]; + int offset; + int index; + + for (int iter = 0; iter < 6; iter++) { + offset = 0; + index = 0; + while (offset < (1<<15)-42) { + buffer.write(&data[offset], dist[d1][index]); + offset += dist[d1][index]; + index++; + } + + offset = 0; + index = 0; + while (offset < (1<<15)-42) { + buffer.read(&data_out[offset], dist[d2][index]); + offset += dist[d2][index]; + index++; + } + + BOOST_CHECK(!memcmp(data, data_out, (1<<15)-42)); + + // Pull out the extra data. + buffer.read(data_out, 42); + } + } + } +} + +BOOST_AUTO_TEST_CASE( test_MemoryBuffer_Write_Read_Multi2 ) { + init_data(); + + // Do shorter writes and reads so we don't align to power-of-two boundaries. + // Pull the buffer out of the loop so its state gets worked harder. + TMemoryBuffer buffer(16); + + for (int d1 = 0; d1 < 3; d1++) { + for (int d2 = 0; d2 < 3; d2++) { + uint8_t data_out[1<<15]; + int offset; + int index; + + for (int iter = 0; iter < 6; iter++) { + offset = 0; + index = 0; + while (offset < (1<<15)-42) { + buffer.write(&data[offset], dist[d1][index]); + offset += dist[d1][index]; + index++; + } + + offset = 0; + index = 0; + while (offset < (1<<15)-42) { + buffer.read(&data_out[offset], dist[d2][index]); + offset += dist[d2][index]; + index++; + } + + BOOST_CHECK(!memcmp(data, data_out, (1<<15)-42)); + + // Pull out the extra data. + buffer.read(data_out, 42); + } + } + } +} + +BOOST_AUTO_TEST_CASE( test_MemoryBuffer_Write_Read_Incomplete ) { + init_data(); + + // Do shorter writes and reads so we don't align to power-of-two boundaries. + // Pull the buffer out of the loop so its state gets worked harder. + + for (int d1 = 0; d1 < 3; d1++) { + for (int d2 = 0; d2 < 3; d2++) { + TMemoryBuffer buffer(16); + uint8_t data_out[1<<13]; + + int write_offset = 0; + int write_index = 0; + unsigned int to_write = (1<<14)-42; + while (to_write > 0) { + int write_amt = std::min(dist[d1][write_index], to_write); + buffer.write(&data[write_offset], write_amt); + write_offset += write_amt; + write_index++; + to_write -= write_amt; + } + + int read_offset = 0; + int read_index = 0; + unsigned int to_read = (1<<13)-42; + while (to_read > 0) { + int read_amt = std::min(dist[d2][read_index], to_read); + int got = buffer.read(&data_out[read_offset], read_amt); + BOOST_CHECK_EQUAL(got, read_amt); + read_offset += read_amt; + read_index++; + to_read -= read_amt; + } + + BOOST_CHECK(!memcmp(data, data_out, (1<<13)-42)); + + int second_offset = write_offset; + int second_index = write_index-1; + unsigned int to_second = (1<<14)+42; + while (to_second > 0) { + int second_amt = std::min(dist[d1][second_index], to_second); + //printf("%d\n", second_amt); + buffer.write(&data[second_offset], second_amt); + second_offset += second_amt; + second_index++; + to_second -= second_amt; + } + + string output = buffer.getBufferAsString(); + BOOST_CHECK_EQUAL(data_str.substr((1<<13)-42), output); + } + } +} + +BOOST_AUTO_TEST_CASE( test_BufferedTransport_Write ) { + init_data(); + + int sizes[] = { + 12, 15, 16, 17, 20, + 501, 512, 523, + 2000, 2048, 2096, + 1<<14, 1<<17, + }; + + foreach (int size, sizes) { + for (int d1 = 0; d1 < 3; d1++) { + shared_ptr buffer(new TMemoryBuffer(16)); + TBufferedTransport trans(buffer, size); + + int offset = 0; + int index = 0; + while (offset < 1<<15) { + trans.write(&data[offset], dist[d1][index]); + offset += dist[d1][index]; + index++; + } + trans.flush(); + + string output = buffer->getBufferAsString(); + BOOST_CHECK_EQUAL(data_str, output); + } + } +} + +BOOST_AUTO_TEST_CASE( test_BufferedTransport_Read_Full ) { + init_data(); + + int sizes[] = { + 12, 15, 16, 17, 20, + 501, 512, 523, + 2000, 2048, 2096, + 1<<14, 1<<17, + }; + + foreach (int size, sizes) { + for (int d1 = 0; d1 < 3; d1++) { + shared_ptr buffer(new TMemoryBuffer(data, sizeof(data))); + TBufferedTransport trans(buffer, size); + uint8_t data_out[1<<15]; + + int offset = 0; + int index = 0; + while (offset < 1<<15) { + // Note: this doesn't work with "read" because TBufferedTransport + // doesn't try loop over reads, so we get short reads. We don't + // check the return value, so that messes us up. + trans.readAll(&data_out[offset], dist[d1][index]); + offset += dist[d1][index]; + index++; + } + + BOOST_CHECK(!memcmp(data, data_out, sizeof(data))); + } + } +} + +BOOST_AUTO_TEST_CASE( test_BufferedTransport_Read_Short ) { + init_data(); + + int sizes[] = { + 12, 15, 16, 17, 20, + 501, 512, 523, + 2000, 2048, 2096, + 1<<14, 1<<17, + }; + + foreach (int size, sizes) { + for (int d1 = 0; d1 < 3; d1++) { + shared_ptr buffer(new TMemoryBuffer(data, sizeof(data))); + shared_ptr tshort(new TShortReadTransport(buffer, 0.125)); + TBufferedTransport trans(buffer, size); + uint8_t data_out[1<<15]; + + int offset = 0; + int index = 0; + while (offset < 1<<15) { + // Note: this doesn't work with "read" because TBufferedTransport + // doesn't try loop over reads, so we get short reads. We don't + // check the return value, so that messes us up. + trans.readAll(&data_out[offset], dist[d1][index]); + offset += dist[d1][index]; + index++; + } + + BOOST_CHECK(!memcmp(data, data_out, sizeof(data))); + } + } +} + +BOOST_AUTO_TEST_CASE( test_FramedTransport_Write ) { + init_data(); + + int sizes[] = { + 12, 15, 16, 17, 20, + 501, 512, 523, + 2000, 2048, 2096, + 1<<14, 1<<17, + }; + + foreach (int size, sizes) { + for (int d1 = 0; d1 < 3; d1++) { + shared_ptr buffer(new TMemoryBuffer(16)); + TFramedTransport trans(buffer, size); + + int offset = 0; + int index = 0; + while (offset < 1<<15) { + trans.write(&data[offset], dist[d1][index]); + offset += dist[d1][index]; + index++; + } + trans.flush(); + + int32_t frame_size = -1; + buffer->read(reinterpret_cast(&frame_size), sizeof(frame_size)); + frame_size = (int32_t)ntohl((uint32_t)frame_size); + BOOST_CHECK_EQUAL(frame_size, 1<<15); + BOOST_CHECK_EQUAL(data_str.size(), (unsigned int)frame_size); + string output = buffer->getBufferAsString(); + BOOST_CHECK_EQUAL(data_str, output); + } + } +} + +BOOST_AUTO_TEST_CASE( test_FramedTransport_Read ) { + init_data(); + + for (int d1 = 0; d1 < 3; d1++) { + uint8_t data_out[1<<15]; + shared_ptr buffer(new TMemoryBuffer()); + TFramedTransport trans(buffer); + int32_t length = sizeof(data); + length = (int32_t)htonl((uint32_t)length); + buffer->write(reinterpret_cast(&length), sizeof(length)); + buffer->write(data, sizeof(data)); + + int offset = 0; + int index = 0; + while (offset < 1<<15) { + // This should work with read because we have one huge frame. + trans.read(&data_out[offset], dist[d1][index]); + offset += dist[d1][index]; + index++; + } + + BOOST_CHECK(!memcmp(data, data_out, sizeof(data))); + } +} + +BOOST_AUTO_TEST_CASE( test_FramedTransport_Write_Read ) { + init_data(); + + int sizes[] = { + 12, 15, 16, 17, 20, + 501, 512, 523, + 2000, 2048, 2096, + 1<<14, 1<<17, + }; + + int probs[] = { 1, 2, 4, 8, 16, 32, }; + + foreach (int size, sizes) { + foreach (int prob, probs) { + for (int d1 = 0; d1 < 3; d1++) { + shared_ptr buffer(new TMemoryBuffer(16)); + TFramedTransport trans(buffer, size); + uint8_t data_out[1<<15]; + std::vector flush_sizes; + + int write_offset = 0; + int write_index = 0; + int flush_size = 0; + while (write_offset < 1<<15) { + trans.write(&data[write_offset], dist[d1][write_index]); + write_offset += dist[d1][write_index]; + flush_size += dist[d1][write_index]; + write_index++; + if (flush_size > 0 && rand()%prob == 0) { + flush_sizes.push_back(flush_size); + flush_size = 0; + trans.flush(); + } + } + if (flush_size != 0) { + flush_sizes.push_back(flush_size); + flush_size = 0; + trans.flush(); + } + + int read_offset = 0; + int read_index = 0; + foreach (int fsize, flush_sizes) { + // We are exploiting an implementation detail of TFramedTransport. + // The read buffer starts empty and it will never do more than one + // readFrame per read, so we should always get exactly one frame. + int got = trans.read(&data_out[read_offset], 1<<15); + BOOST_CHECK_EQUAL(got, fsize); + read_offset += got; + read_index++; + } + + BOOST_CHECK_EQUAL((unsigned int)read_offset, sizeof(data)); + BOOST_CHECK(!memcmp(data, data_out, sizeof(data))); + } + } + } +} + +BOOST_AUTO_TEST_CASE( test_FramedTransport_Empty_Flush ) { + init_data(); + + string output1("\x00\x00\x00\x01""a", 5); + string output2("\x00\x00\x00\x01""a\x00\x00\x00\x02""bc", 11); + + shared_ptr buffer(new TMemoryBuffer()); + TFramedTransport trans(buffer); + + BOOST_CHECK_EQUAL(buffer->getBufferAsString(), ""); + trans.flush(); + BOOST_CHECK_EQUAL(buffer->getBufferAsString(), ""); + trans.flush(); + trans.flush(); + BOOST_CHECK_EQUAL(buffer->getBufferAsString(), ""); + trans.write((const uint8_t*)"a", 1); + BOOST_CHECK_EQUAL(buffer->getBufferAsString(), ""); + trans.flush(); + BOOST_CHECK_EQUAL(buffer->getBufferAsString(), output1); + trans.flush(); + trans.flush(); + BOOST_CHECK_EQUAL(buffer->getBufferAsString(), output1); + trans.write((const uint8_t*)"bc", 2); + BOOST_CHECK_EQUAL(buffer->getBufferAsString(), output1); + trans.flush(); + BOOST_CHECK_EQUAL(buffer->getBufferAsString(), output2); + trans.flush(); + trans.flush(); + BOOST_CHECK_EQUAL(buffer->getBufferAsString(), output2); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/test/TFDTransportTest.cpp b/test/TFDTransportTest.cpp new file mode 100644 index 00000000..1ec538e3 --- /dev/null +++ b/test/TFDTransportTest.cpp @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +using apache::thrift::transport::TTransportException; +using apache::thrift::transport::TFDTransport; + +class DummyException : std::exception { +}; + +int main() { + { + TFDTransport t(256, TFDTransport::NO_CLOSE_ON_DESTROY); + } + + try { + { + TFDTransport t(256, TFDTransport::CLOSE_ON_DESTROY); + } + std::abort(); + } catch (TTransportException) { + } + + try { + { + TFDTransport t(256, TFDTransport::CLOSE_ON_DESTROY); + throw DummyException(); + } + std::abort(); + } catch (TTransportException&) { + abort(); + } catch (DummyException&) { + } + + return 0; + +} diff --git a/test/TMemoryBufferTest.cpp b/test/TMemoryBufferTest.cpp new file mode 100644 index 00000000..49bd10b5 --- /dev/null +++ b/test/TMemoryBufferTest.cpp @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include "gen-cpp/ThriftTest_types.h" + +BOOST_AUTO_TEST_SUITE( TMemoryBufferTest ) + +BOOST_AUTO_TEST_CASE( test_roundtrip ) { + using apache::thrift::transport::TMemoryBuffer; + using apache::thrift::protocol::TBinaryProtocol; + using boost::shared_ptr; + + shared_ptr strBuffer(new TMemoryBuffer()); + shared_ptr binaryProtcol(new TBinaryProtocol(strBuffer)); + + thrift::test::Xtruct a; + a.i32_thing = 10; + a.i64_thing = 30; + a.string_thing ="holla back a"; + + a.write(binaryProtcol.get()); + std::string serialized = strBuffer->getBufferAsString(); + + shared_ptr strBuffer2(new TMemoryBuffer()); + shared_ptr binaryProtcol2(new TBinaryProtocol(strBuffer2)); + + strBuffer2->resetBuffer((uint8_t*)serialized.data(), serialized.length()); + thrift::test::Xtruct a2; + a2.read(binaryProtcol2.get()); + + assert(a == a2); + } + +BOOST_AUTO_TEST_CASE( test_copy ) + { + using apache::thrift::transport::TMemoryBuffer; + using std::string; + using std::cout; + using std::endl; + + string* str1 = new string("abcd1234"); + const char* data1 = str1->data(); + TMemoryBuffer buf((uint8_t*)str1->data(), str1->length(), TMemoryBuffer::COPY); + delete str1; + string* str2 = new string("plsreuse"); + bool obj_reuse = (str1 == str2); + bool dat_reuse = (data1 == str2->data()); + cout << "Object reuse: " << obj_reuse << " Data reuse: " << dat_reuse + << ((obj_reuse && dat_reuse) ? " YAY!" : "") << endl; + delete str2; + + string str3 = "wxyz", str4 = "6789"; + buf.readAppendToString(str3, 4); + buf.readAppendToString(str4, INT_MAX); + + assert(str3 == "wxyzabcd"); + assert(str4 == "67891234"); + } + +BOOST_AUTO_TEST_CASE( test_exceptions ) + { + using apache::thrift::transport::TTransportException; + using apache::thrift::transport::TMemoryBuffer; + using std::string; + + char data[] = "foo\0bar"; + + TMemoryBuffer buf1((uint8_t*)data, 7, TMemoryBuffer::OBSERVE); + string str = buf1.getBufferAsString(); + assert(str.length() == 7); + buf1.resetBuffer(); + try { + buf1.write((const uint8_t*)"foo", 3); + assert(false); + } catch (TTransportException& ex) {} + + TMemoryBuffer buf2((uint8_t*)data, 7, TMemoryBuffer::COPY); + try { + buf2.write((const uint8_t*)"bar", 3); + } catch (TTransportException& ex) { + assert(false); + } + } + +BOOST_AUTO_TEST_SUITE_END() diff --git a/test/TPipedTransportTest.cpp b/test/TPipedTransportTest.cpp new file mode 100644 index 00000000..5708fd21 --- /dev/null +++ b/test/TPipedTransportTest.cpp @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +using namespace std; +using boost::shared_ptr; +using apache::thrift::transport::TTransportException; +using apache::thrift::transport::TPipedTransport; +using apache::thrift::transport::TMemoryBuffer; + +int main() { + shared_ptr underlying(new TMemoryBuffer); + shared_ptr pipe(new TMemoryBuffer); + shared_ptr trans(new TPipedTransport(underlying, pipe)); + + uint8_t buffer[4]; + + underlying->write((uint8_t*)"abcd", 4); + trans->readAll(buffer, 2); + assert( string((char*)buffer, 2) == "ab" ); + trans->readEnd(); + assert( pipe->getBufferAsString() == "ab" ); + pipe->resetBuffer(); + underlying->write((uint8_t*)"ef", 2); + trans->readAll(buffer, 2); + assert( string((char*)buffer, 2) == "cd" ); + trans->readAll(buffer, 2); + assert( string((char*)buffer, 2) == "ef" ); + trans->readEnd(); + assert( pipe->getBufferAsString() == "cdef" ); + + return 0; + +} diff --git a/test/ThriftTest.thrift b/test/ThriftTest.thrift new file mode 100644 index 00000000..3517640a --- /dev/null +++ b/test/ThriftTest.thrift @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace java thrift.test +namespace cpp thrift.test +namespace rb Thrift.Test +namespace perl ThriftTest +namespace csharp Thrift.Test + +enum Numberz +{ + ONE = 1, + TWO, + THREE, + FIVE = 5, + SIX, + EIGHT = 8 +} + +typedef i64 UserId + +struct Bonk +{ + 1: string message, + 2: i32 type +} + +struct Bools { + 1: bool im_true, + 2: bool im_false, +} + +struct Xtruct +{ + 1: string string_thing, + 4: byte byte_thing, + 9: i32 i32_thing, + 11: i64 i64_thing +} + +struct Xtruct2 +{ + 1: byte byte_thing, + 2: Xtruct struct_thing, + 3: i32 i32_thing +} + +struct Xtruct3 +{ + 1: string string_thing, + 4: i32 changed, + 9: i32 i32_thing, + 11: i64 i64_thing +} + + +struct Insanity +{ + 1: map userMap, + 2: list xtructs +} + +struct CrazyNesting { + 1: string string_field, + 2: optional set set_field, + 3: required list< map,map>>>>> list_field +} + +exception Xception { + 1: i32 errorCode, + 2: string message +} + +exception Xception2 { + 1: i32 errorCode, + 2: Xtruct struct_thing +} + +struct EmptyStruct {} + +struct OneField { + 1: EmptyStruct field +} + +service ThriftTest +{ + void testVoid(), + string testString(1: string thing), + byte testByte(1: byte thing), + i32 testI32(1: i32 thing), + i64 testI64(1: i64 thing), + double testDouble(1: double thing), + Xtruct testStruct(1: Xtruct thing), + Xtruct2 testNest(1: Xtruct2 thing), + map testMap(1: map thing), + set testSet(1: set thing), + list testList(1: list thing), + Numberz testEnum(1: Numberz thing), + UserId testTypedef(1: UserId thing), + + map> testMapMap(1: i32 hello), + + /* So you think you've got this all worked, out eh? */ + map> testInsanity(1: Insanity argument), + + /* Multiple parameters */ + Xtruct testMulti(1: byte arg0, 2: i32 arg1, 3: i64 arg2, 4: map arg3, 5: Numberz arg4, 6: UserId arg5), + + /* Exception specifier */ + + void testException(1: string arg) throws(1: Xception err1), + + /* Multiple exceptions specifier */ + + Xtruct testMultiException(1: string arg0, 2: string arg1) throws(1: Xception err1, 2: Xception2 err2) + + /* Test oneway void */ + oneway void testOneway(1:i32 secondsToSleep) +} + +service SecondService +{ + void blahBlah() +} + +struct VersioningTestV1 { + 1: i32 begin_in_both, + 3: string old_string, + 12: i32 end_in_both +} + +struct VersioningTestV2 { + 1: i32 begin_in_both, + + 2: i32 newint, + 3: byte newbyte, + 4: i16 newshort, + 5: i64 newlong, + 6: double newdouble + 7: Bonk newstruct, + 8: list newlist, + 9: set newset, + 10: map newmap, + 11: string newstring, + 12: i32 end_in_both +} + +struct ListTypeVersioningV1 { + 1: list myints; + 2: string hello; +} + +struct ListTypeVersioningV2 { + 1: list strings; + 2: string hello; +} diff --git a/test/ThriftTest_extras.cpp b/test/ThriftTest_extras.cpp new file mode 100644 index 00000000..b78f2763 --- /dev/null +++ b/test/ThriftTest_extras.cpp @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Extra functions required for ThriftTest_types to work + +#include +#include "gen-cpp/ThriftTest_types.h" + + +namespace thrift { namespace test { + +bool Insanity::operator<(thrift::test::Insanity const& other) const { + using apache::thrift::ThriftDebugString; + return ThriftDebugString(*this) < ThriftDebugString(other); +} + +}} diff --git a/test/UnitTestMain.cpp b/test/UnitTestMain.cpp new file mode 100644 index 00000000..d90c54f4 --- /dev/null +++ b/test/UnitTestMain.cpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#define BOOST_TEST_MODULE thrift +#include diff --git a/test/ZlibTest.cpp b/test/ZlibTest.cpp new file mode 100644 index 00000000..45d3ecc4 --- /dev/null +++ b/test/ZlibTest.cpp @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* +thrift --gen cpp DebugProtoTest.thrift +g++ -Wall -g -I../lib/cpp/src -I/usr/local/include/boost-1_33_1 \ + ZlibTest.cpp \ + ../lib/cpp/.libs/libthriftz.a ../lib/cpp/.libs/libthrift.a \ + -lz -o ZlibTest +./ZlibTest +*/ + +#include +#include +#include +#include +#include +#include + + +// Distributions of reads and writes meant to approximate a real load, +// mixing up small and large while also hitting various boundary conditions. +// Generated by Python: int(random.lognormvariate(2.5, 1)) +unsigned int dist[][5000] = { + { 1<<15 }, + + { + 5,13,9,1,8,9,11,13,18,48,24,13,21,13,5,11,35,2,4,20,17,72,27,14,15,4,7,26, + 12,1,14,9,2,16,29,41,7,24,4,27,14,4,1,4,25,3,6,34,10,8,50,2,14,13,55,29,3, + 43,53,49,14,4,10,32,27,48,1,3,1,11,5,17,16,51,17,30,15,11,9,2,2,11,52,12,2, + 13,94,1,19,1,38,2,8,43,8,33,7,30,8,17,22,2,15,14,12,34,2,12,6,37,29,74,3, + 165,16,11,17,5,14,3,10,7,37,11,24,7,1,3,12,37,8,9,34,17,12,8,21,13,37,1,4, + 30,14,78,4,15,2,40,37,17,12,36,82,14,4,1,4,7,17,11,16,88,77,2,3,15,3,34,11, + 5,79,22,34,8,4,4,40,22,24,28,9,13,3,34,27,9,16,39,16,39,13,2,4,3,41,26,10,4, + 33,4,7,12,5,6,3,10,30,8,21,16,58,19,9,0,47,7,13,11,19,15,7,53,57,2,13,28,22, + 3,16,9,25,33,12,40,7,12,64,7,14,24,44,9,2,14,11,2,58,1,26,30,11,9,5,24,7,9, + 94,2,10,21,5,5,4,5,6,179,9,18,2,7,13,31,41,17,4,36,3,21,6,26,8,15,18,44,27, + 11,9,25,7,0,14,2,12,20,23,13,2,163,9,5,15,65,2,14,6,8,98,11,15,14,34,2,3,10, + 22,9,92,7,10,32,67,13,3,4,35,8,2,1,5,0,26,381,7,27,8,2,16,93,4,19,5,8,25,9, + 31,14,4,21,5,3,9,22,56,4,18,3,11,18,6,4,3,40,12,16,110,8,35,14,1,18,40,9,12, + 14,3,11,7,57,13,18,116,53,19,22,7,16,11,5,8,21,16,1,75,21,20,1,28,2,6,1,7, + 19,38,5,6,9,9,4,1,7,55,36,62,5,4,4,24,15,1,12,35,48,20,5,17,1,5,26,15,4,54, + 13,5,5,15,5,19,32,29,31,7,6,40,7,80,11,18,8,128,48,6,12,84,13,4,7,2,13,9,16, + 17,3,254,1,4,181,8,44,7,6,24,27,9,23,14,34,16,22,25,10,3,3,4,4,12,2,12,6,7, + 13,58,13,6,11,19,53,11,66,18,19,10,4,13,2,5,49,58,1,67,7,21,64,14,11,14,8,3, + 26,33,91,31,20,7,9,42,39,4,3,55,11,10,0,7,4,75,8,12,0,27,3,8,9,0,12,12,23, + 28,23,20,4,13,30,2,22,20,19,30,6,22,2,6,4,24,7,19,55,86,5,33,2,161,6,7,1,62, + 13,3,72,12,12,9,7,12,10,5,10,29,1,5,22,13,13,5,2,12,3,7,14,18,2,3,46,21,17, + 15,19,3,27,5,16,45,31,10,8,17,18,18,3,7,24,6,55,9,3,6,12,10,12,8,91,9,4,4,4, + 27,29,16,5,7,22,43,28,11,14,8,11,28,109,55,71,40,3,8,22,26,15,44,3,25,29,5, + 3,32,17,12,3,29,27,25,15,11,8,40,39,38,17,3,9,11,2,32,11,6,20,48,75,27,3,7, + 54,12,95,12,7,24,23,2,13,8,15,16,5,12,4,17,7,19,88,2,6,13,115,45,12,21,2,86, + 74,9,7,5,16,32,16,2,21,18,6,34,5,18,260,7,12,16,44,19,92,31,7,8,2,9,0,0,15, + 8,38,4,8,20,18,2,83,3,3,4,9,5,3,10,3,5,29,15,7,11,8,48,17,23,2,17,4,11,22, + 21,64,8,8,4,19,95,0,17,28,9,11,20,71,5,11,18,12,13,45,49,4,1,33,32,23,13,5, + 52,2,2,16,3,4,7,12,2,1,12,6,24,1,22,155,21,3,45,4,12,44,26,5,40,36,9,9,8,20, + 35,31,3,2,32,50,10,8,37,2,75,35,22,15,192,8,11,23,1,4,29,6,8,8,5,12,18,32,4, + 7,12,2,0,0,9,5,48,11,35,3,1,123,6,29,8,11,8,23,51,16,6,63,12,2,5,4,14,2,15, + 7,14,3,2,7,17,32,8,8,10,1,23,62,2,49,6,49,47,23,3,20,7,11,39,10,24,6,15,5,5, + 11,8,16,36,8,13,20,3,10,44,7,52,7,10,36,6,15,10,5,11,4,14,19,17,10,12,3,6, + 23,4,13,94,70,7,36,7,38,7,28,8,4,15,3,19,4,33,39,21,109,4,80,6,40,4,432,4,4, + 7,8,3,31,8,28,37,34,10,2,21,5,22,0,7,36,14,12,6,24,1,21,5,9,2,29,20,54,113, + 13,31,39,27,6,0,27,4,5,2,43,7,8,57,8,62,7,9,12,22,90,30,6,19,7,10,20,6,5,58, + 32,30,41,4,10,25,13,3,8,7,10,2,9,6,151,44,16,12,16,20,8,3,18,11,17,4,10,45, + 15,8,56,38,52,25,40,14,4,17,15,8,2,19,7,8,26,30,2,3,180,8,26,17,38,35,5,16, + 28,5,15,56,13,14,18,9,15,83,27,3,9,4,11,8,27,27,44,10,12,8,3,48,14,7,9,4,4, + 8,4,5,9,122,8,14,12,19,17,21,4,29,63,21,17,10,12,18,47,10,10,53,4,18,16,4,8, + 118,9,5,12,9,11,9,3,12,32,3,23,2,15,3,3,30,3,17,235,15,22,9,299,14,17,1,5, + 16,8,3,7,3,13,2,7,6,4,8,66,2,13,6,15,16,47,3,36,5,7,10,24,1,9,9,8,13,16,26, + 12,7,24,21,18,49,23,39,10,41,4,13,4,27,11,12,12,19,4,147,8,10,9,40,21,2,83, + 10,5,6,11,25,9,50,57,40,12,12,21,1,3,24,23,9,3,9,13,2,3,12,57,8,11,13,15,26, + 15,10,47,36,4,25,1,5,8,5,4,0,12,49,5,19,4,6,16,14,6,10,69,10,33,29,7,8,61, + 12,4,0,3,7,6,3,16,29,27,38,4,21,0,24,3,2,1,19,16,22,2,8,138,11,7,7,3,12,22, + 3,16,5,7,3,53,9,10,32,14,5,7,3,6,22,9,59,26,8,7,58,5,16,11,55,7,4,11,146,91, + 8,13,18,14,6,8,8,31,26,22,6,11,30,11,30,15,18,31,3,48,17,7,6,4,9,2,25,3,35, + 13,13,7,8,4,31,10,8,10,4,3,45,10,23,2,7,259,17,21,13,14,3,26,3,8,27,4,18,9, + 66,7,12,5,8,17,4,23,55,41,51,2,32,26,66,4,21,14,12,65,16,22,17,5,14,2,29,24, + 7,3,36,2,43,53,86,5,28,4,58,13,49,121,6,2,73,2,1,47,4,2,27,10,35,28,27,10, + 17,10,56,7,10,14,28,20,24,40,7,4,7,3,10,11,32,6,6,3,15,11,54,573,2,3,6,2,3, + 14,64,4,16,12,16,42,10,26,4,6,11,69,18,27,2,2,17,22,9,13,22,11,6,1,15,49,3, + 14,1 + }, + + { + 11,11,11,15,47,1,3,1,23,5,8,18,3,23,15,21,1,7,19,10,26,1,17,11,31,21,41,18, + 34,4,9,58,19,3,3,36,5,18,13,3,14,4,9,10,4,19,56,15,3,5,3,11,27,9,4,10,13,4, + 11,6,9,2,18,3,10,19,11,4,53,4,2,2,3,4,58,16,3,0,5,30,2,11,93,10,2,14,10,6,2, + 115,2,25,16,22,38,101,4,18,13,2,145,51,45,15,14,15,13,20,7,24,5,13,14,30,40, + 10,4,107,12,24,14,39,12,6,13,20,7,7,11,5,18,18,45,22,6,39,3,2,1,51,9,11,4, + 13,9,38,44,8,11,9,15,19,9,23,17,17,17,13,9,9,1,10,4,18,6,2,9,5,27,32,72,8, + 37,9,4,10,30,17,20,15,17,66,10,4,73,35,37,6,4,16,117,45,13,4,75,5,24,65,10, + 4,9,4,13,46,5,26,29,10,4,4,52,3,13,18,63,6,14,9,24,277,9,88,2,48,27,123,14, + 61,7,5,10,8,7,90,3,10,3,3,48,17,13,10,18,33,2,19,36,6,21,1,16,12,5,6,2,16, + 15,29,88,28,2,15,6,11,4,6,11,3,3,4,18,9,53,5,4,3,33,8,9,8,6,7,36,9,62,14,2, + 1,10,1,16,7,32,7,23,20,11,10,23,2,1,0,9,16,40,2,81,5,22,8,5,4,37,51,37,10, + 19,57,11,2,92,31,6,39,10,13,16,8,20,6,9,3,10,18,25,23,12,30,6,2,26,7,64,18, + 6,30,12,13,27,7,10,5,3,33,24,99,4,23,4,1,27,7,27,49,8,20,16,3,4,13,9,22,67, + 28,3,10,16,3,2,10,4,8,1,8,19,3,85,6,21,1,9,16,2,30,10,33,12,4,9,3,1,60,38,6, + 24,32,3,14,3,40,8,34,115,5,9,27,5,96,3,40,6,15,5,8,22,112,5,5,25,17,58,2,7, + 36,21,52,1,3,95,12,21,4,11,8,59,24,5,21,4,9,15,8,7,21,3,26,5,11,6,7,17,65, + 14,11,10,2,17,5,12,22,4,4,2,21,8,112,3,34,63,35,2,25,1,2,15,65,23,0,3,5,15, + 26,27,9,5,48,11,15,4,9,5,33,20,15,1,18,19,11,24,40,10,21,74,6,6,32,30,40,5, + 4,7,44,10,25,46,16,12,5,40,7,18,5,18,9,12,8,4,25,5,6,36,4,43,8,9,12,35,17,4, + 8,9,11,27,5,10,17,40,8,12,4,18,9,18,12,20,25,39,42,1,24,13,22,15,7,112,35,3, + 7,17,33,2,5,5,19,8,4,12,24,14,13,2,1,13,6,5,19,11,7,57,0,19,6,117,48,14,8, + 10,51,17,12,14,2,5,8,9,15,4,48,53,13,22,4,25,12,11,19,45,5,2,6,54,22,9,15,9, + 13,2,7,11,29,82,16,46,4,26,14,26,40,22,4,26,6,18,13,4,4,20,3,3,7,12,17,8,9, + 23,6,20,7,25,23,19,5,15,6,23,15,11,19,11,3,17,59,8,18,41,4,54,23,44,75,13, + 20,6,11,2,3,1,13,10,3,7,12,3,4,7,8,30,6,6,7,3,32,9,5,28,6,114,42,13,36,27, + 59,6,93,13,74,8,69,140,3,1,17,48,105,6,11,5,15,1,10,10,14,8,53,0,8,24,60,2, + 6,35,2,12,32,47,16,17,75,2,5,4,37,28,10,5,9,57,4,59,5,12,13,7,90,5,11,5,24, + 22,13,30,1,2,10,9,6,19,3,18,47,2,5,7,9,35,15,3,6,1,21,14,14,18,14,9,12,8,73, + 6,19,3,32,9,14,17,17,5,55,23,6,16,28,3,11,48,4,6,6,6,12,16,30,10,30,27,51, + 18,29,2,3,15,1,76,0,16,33,4,27,3,62,4,10,2,4,8,15,9,41,26,22,2,4,20,4,49,0, + 8,1,57,13,12,39,3,63,10,19,34,35,2,7,8,29,72,4,10,0,77,8,6,7,9,15,21,9,4,1, + 20,23,1,9,18,9,15,36,4,7,6,15,5,7,7,40,2,9,22,2,3,20,4,12,34,13,6,18,15,1, + 38,20,12,7,16,3,19,85,12,16,18,16,2,17,1,13,8,6,12,15,97,17,12,9,3,21,15,12, + 23,44,81,26,30,2,5,17,6,6,0,22,42,19,6,19,41,14,36,7,3,56,7,9,3,2,6,9,69,3, + 15,4,30,28,29,7,9,15,17,17,6,1,6,153,9,33,5,12,14,16,28,3,8,7,14,12,4,6,36, + 9,24,13,13,4,2,9,15,19,9,53,7,13,4,150,17,9,2,6,12,7,3,5,58,19,58,28,8,14,3, + 20,3,0,32,56,7,5,4,27,1,68,4,29,13,5,58,2,9,65,41,27,16,15,12,14,2,10,9,24, + 3,2,9,2,2,3,14,32,10,22,3,13,11,4,6,39,17,0,10,5,5,10,35,16,19,14,1,8,63,19, + 14,8,56,10,2,12,6,12,6,7,16,2,9,9,12,20,73,25,13,21,17,24,5,32,8,12,25,8,14, + 16,5,23,3,7,6,3,11,24,6,30,4,21,13,28,4,6,29,15,5,17,6,26,8,15,8,3,7,7,50, + 11,30,6,2,28,56,16,24,25,23,24,89,31,31,12,7,22,4,10,17,3,3,8,11,13,5,3,27, + 1,12,1,14,8,10,29,2,5,2,2,20,10,0,31,10,21,1,48,3,5,43,4,5,18,13,5,18,25,34, + 18,3,5,22,16,3,4,20,3,9,3,25,6,6,44,21,3,12,7,5,42,3,2,14,4,36,5,3,45,51,15, + 9,11,28,9,7,6,6,12,26,5,14,10,11,42,55,13,21,4,28,6,7,23,27,11,1,41,36,0,32, + 15,26,2,3,23,32,11,2,15,7,29,26,144,33,20,12,7,21,10,7,11,65,46,10,13,20,32, + 4,4,5,19,2,19,15,49,41,1,75,10,11,25,1,2,45,11,8,27,18,10,60,28,29,12,30,19, + 16,4,24,11,19,27,17,49,18,7,40,13,19,22,8,55,12,11,3,6,5,11,8,10,22,5,9,9, + 25,7,17,7,64,1,24,2,12,17,44,4,12,27,21,11,10,7,47,5,9,13,12,38,27,21,7,29, + 7,1,17,3,3,5,48,62,10,3,11,17,15,15,6,3,8,10,8,18,19,13,3,9,7,6,44,9,10,4, + 43,8,6,6,14,20,38,24,2,4,5,5,7,5,9,39,8,44,40,9,19,7,3,15,25,2,37,18,15,9,5, + 8,32,10,5,18,4,7,46,20,17,23,4,11,16,18,31,11,3,11,1,14,1,25,4,27,13,13,39, + 14,6,6,35,6,16,13,11,122,21,15,20,24,10,5,152,15,39,5,20,16,9,14,7,53,6,3,8, + 19,63,32,6,2,3,20,1,19,5,13,42,15,4,6,68,31,46,11,38,10,24,5,5,8,9,12,3,35, + 46,26,16,2,8,4,74,16,44,4,5,1,16,4,14,23,16,69,15,42,31,14,7,7,6,97,14,40,1, + 8,7,34,9,39,19,13,15,10,21,18,10,5,15,38,7,5,12,7,20,15,4,11,6,14,5,17,7,39, + 35,36,18,20,26,22,4,2,36,21,64,0,5,9,10,6,4,1,7,3,1,3,3,4,10,20,90,2,22,48, + 16,23,2,33,40,1,21,21,17,20,8,8,12,4,83,14,48,4,21,3,9,27,5,11,40,15,9,3,16, + 17,9,11,4,24,31,17,3,4,2,11,1,8,4,8,6,41,17,4,13,3,7,17,8,27,5,13,6,10,7,13, + 12,18,13,60,18,3,8,1,12,125,2,7,16,2,11,2,4,7,26,5,9,14,14,16,8,14,7,14,6,9, + 13,9,6,4,26,35,49,36,55,3,9,6,40,26,23,31,19,41,2,10,31,6,54,5,69,16,7,8,16, + 1,5,7,4,22,7,7,5,4,48,11,13,3,98,4,11,19,4,2,14,7,34,7,10,3,2,12,7,6,2,5,118 + }, +}; + + +int main() { + using namespace std; + using namespace boost; + using namespace apache::thrift::transport; + + char *file_names[] = { + // Highly compressible. + "./gen-cpp/DebugProtoTest_types.cpp", + // Uncompressible. + "/dev/urandom", + // Null-terminated. + NULL, + }; + + + for (char** fnamep = &file_names[0]; *fnamep != NULL; fnamep++) { + ifstream file(*fnamep); + char buf[32*1024]; + file.read(buf, sizeof(buf)); + vector content(buf, buf+file.gcount()); + vector mirror; + file.close(); + + assert(content.size() == 32*1024); + + // Let's just start with the big dog! + { + mirror.clear(); + shared_ptr membuf(new TMemoryBuffer()); + shared_ptr zlib_trans(new TZlibTransport(membuf, false)); + zlib_trans->write(&content[0], content.size()); + zlib_trans->flush(); + mirror.resize(content.size()); + uint32_t got = zlib_trans->read(&mirror[0], mirror.size()); + assert(got == content.size()); + assert(mirror == content); + zlib_trans->verifyChecksum(); + } + + // This one is tricky. I separate the last byte of the stream out + // into a separate crbuf_. The last byte is part of the checksum, + // so the entire read goes fine, but when I go to verify the checksum + // it isn't there. The original implementation complained that + // the stream was not complete. I'm about to go fix that. + // It worked. Awesome. + { + mirror.clear(); + shared_ptr membuf(new TMemoryBuffer()); + shared_ptr zlib_trans(new TZlibTransport(membuf, false)); + zlib_trans->write(&content[0], content.size()); + zlib_trans->flush(); + string tmp_buf; + membuf->appendBufferToString(tmp_buf); + zlib_trans.reset(new TZlibTransport(membuf, false, + TZlibTransport::DEFAULT_URBUF_SIZE, + tmp_buf.length()-1)); + mirror.resize(content.size()); + uint32_t got = zlib_trans->read(&mirror[0], mirror.size()); + assert(got == content.size()); + assert(mirror == content); + zlib_trans->verifyChecksum(); + } + + // Make sure we still get that "not complete" error if + // it really isn't complete. + { + mirror.clear(); + shared_ptr membuf(new TMemoryBuffer()); + shared_ptr zlib_trans(new TZlibTransport(membuf, false)); + zlib_trans->write(&content[0], content.size()); + zlib_trans->flush(); + string tmp_buf; + membuf->appendBufferToString(tmp_buf); + tmp_buf.erase(tmp_buf.length() - 1); + membuf->resetFromString(tmp_buf); + mirror.resize(content.size()); + uint32_t got = zlib_trans->read(&mirror[0], mirror.size()); + assert(got == content.size()); + assert(mirror == content); + try { + zlib_trans->verifyChecksum(); + assert(false); + } catch (TTransportException& ex) { + assert(ex.getType() == TTransportException::CORRUPTED_DATA); + } + } + + // Try it with a mix of read/write sizes. + for (int d1 = 0; d1 < 3; d1++) { + for (int d2 = 0; d2 < 3; d2++) { + mirror.clear(); + shared_ptr membuf(new TMemoryBuffer()); + shared_ptr zlib_trans(new TZlibTransport(membuf, false)); + int idx; + unsigned int tot; + + idx = 0; + tot = 0; + while (tot < content.size()) { + zlib_trans->write(&content[tot], dist[d1][idx]); + tot += dist[d1][idx]; + idx++; + } + + zlib_trans->flush(); + mirror.resize(content.size()); + + idx = 0; + tot = 0; + while (tot < mirror.size()) { + uint32_t got = zlib_trans->read(&mirror[tot], dist[d2][idx]); + assert(got == dist[d2][idx]); + tot += dist[d2][idx]; + idx++; + } + + assert(mirror == content); + zlib_trans->verifyChecksum(); + } + } + + // Verify checksum checking. + { + mirror.clear(); + shared_ptr membuf(new TMemoryBuffer()); + shared_ptr zlib_trans(new TZlibTransport(membuf, false)); + zlib_trans->write(&content[0], content.size()); + zlib_trans->flush(); + string tmp_buf; + membuf->appendBufferToString(tmp_buf); + tmp_buf[57]++; + membuf->resetFromString(tmp_buf); + mirror.resize(content.size()); + try { + zlib_trans->read(&mirror[0], mirror.size()); + zlib_trans->verifyChecksum(); + assert(false); + } catch (TZlibTransportException& ex) { + assert(ex.getType() == TTransportException::INTERNAL_ERROR); + } + } + } + + return 0; +} diff --git a/test/cpp/Stress-test.mk b/test/cpp/Stress-test.mk new file mode 100644 index 00000000..1a753b3a --- /dev/null +++ b/test/cpp/Stress-test.mk @@ -0,0 +1,71 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +ifndef thrift_home +thrift_home=../.. +endif #thrift_home + +target: all + +ifndef boost_home +#boost_home=../../../../../thirdparty/boost_1_33_1 +boost_home=/usr/local/include/boost-1_33_1 +endif #boost_home +target: all + +include_paths = $(thrift_home)/lib/cpp/src \ + $(thrift_home)/lib/cpp \ + $(thrift_home)/ \ + $(boost_home) + +include_flags = $(patsubst %,-I%, $(include_paths)) + +# Tools +ifndef THRIFT +THRIFT = ../../compiler/cpp/thrift +endif # THRIFT + +CC = g++ +LD = g++ + +# Compiler flags +DCFL = -Wall -O3 -g -I./gen-cpp $(include_flags) -L$(thrift_home)/lib/cpp/.libs -lthrift -lthriftnb -levent +CFL = -Wall -O3 -I./gen-cpp $(include_flags) -L$(thrift_home)/lib/cpp/.libs -lthrift -lthriftnb -levent + +all: stress-test stress-test-nb + +debug: stress-test-debug stress-test-debug-nb + +stubs: ../StressTest.thrift + $(THRIFT) --gen cpp --gen php ../StressTest.thrift + +stress-test-debug-nb: stubs + g++ -o stress-test-nb $(DCFL) src/nb-main.cpp ./gen-cpp/Service.cpp gen-cpp/StressTest_types.cpp + +stress-test-nb: stubs + g++ -o stress-test-nb $(CFL) src/nb-main.cpp ./gen-cpp/Service.cpp gen-cpp/StressTest_types.cpp + +stress-test-debug: stubs + g++ -o stress-test $(DCFL) src/main.cpp ./gen-cpp/Service.cpp gen-cpp/StressTest_types.cpp + +stress-test: stubs + g++ -o stress-test $(CFL) src/main.cpp ./gen-cpp/Service.cpp gen-cpp/StressTest_types.cpp + +clean: + rm -fr stress-test stress-test-nb gen-cpp diff --git a/test/cpp/Thrift-test.mk b/test/cpp/Thrift-test.mk new file mode 100644 index 00000000..fb3e38bc --- /dev/null +++ b/test/cpp/Thrift-test.mk @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Makefile for Thrift test project. +# Default target is everything + +ifndef thrift_home +thrift_home=../.. +endif #thrift_home + +target: all + +ifndef boost_home +#boost_home=../../../../../thirdparty/boost_1_33_1 +boost_home=/usr/local/include/boost-1_33_1 +endif #boost_home +target: all + +include_paths = $(thrift_home)/lib/cpp/src \ + $(boost_home) + +include_flags = $(patsubst %,-I%, $(include_paths)) + +# Tools +ifndef THRIFT +THRIFT = ../../compiler/cpp/thrift +endif # THRIFT + +CC = g++ +LD = g++ + +# Compiler flags +DCFL = -Wall -O3 -g -I. -I./gen-cpp $(include_flags) -L$(thrift_home)/lib/cpp/.libs -lthrift -lthriftnb -levent +LFL = -L$(thrift_home)/lib/cpp/.libs -lthrift -lthriftnb -levent +CCFL = -Wall -O3 -I. -I./gen-cpp $(include_flags) +CFL = $(CCFL) $(LFL) + +all: server client + +debug: server-debug client-debug + +stubs: ../ThriftTest.thrift + $(THRIFT) --gen cpp ../ThriftTest.thrift + +server-debug: stubs + g++ -o TestServer $(DCFL) src/TestServer.cpp ./gen-cpp/ThriftTest.cpp ./gen-cpp/ThriftTest_types.cpp ../ThriftTest_extras.cpp + +client-debug: stubs + g++ -o TestClient $(DCFL) src/TestClient.cpp ./gen-cpp/ThriftTest.cpp ./gen-cpp/ThriftTest_types.cpp ../ThriftTest_extras.cpp + +server: stubs + g++ -o TestServer $(CFL) src/TestServer.cpp ./gen-cpp/ThriftTest.cpp ./gen-cpp/ThriftTest_types.cpp ../ThriftTest_extras.cpp + +client: stubs + g++ -o TestClient $(CFL) src/TestClient.cpp ./gen-cpp/ThriftTest.cpp ./gen-cpp/ThriftTest_types.cpp ../ThriftTest_extras.cpp + +small: + $(THRIFT) --gen cpp ../SmallTest.thrift + g++ -c $(CCFL) ./gen-cpp/SmallService.cpp ./gen-cpp/SmallTest_types.cpp + +clean: + rm -fr *.o TestServer TestClient gen-cpp diff --git a/test/cpp/realloc/Makefile b/test/cpp/realloc/Makefile new file mode 100644 index 00000000..f89bbb3c --- /dev/null +++ b/test/cpp/realloc/Makefile @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# This probably should not go into "make check", because it is an experiment, +# not a test. Specifically, it is meant to determine how likely realloc is +# to avoid a copy. This is poorly documented. + +run: realloc_test + for it in 1 4 64 ; do \ + for nb in 1 8 64 512 ; do \ + for mins in 64 512 ; do \ + for maxs in 2048 262144 ; do \ + for db in 8 64 ; do \ + ./realloc_test $$nb $$mins $$maxs $$db $$it \ + ; done \ + ; done \ + ; done \ + ; done \ + ; done \ + > raw_stats + +CFLAGS = -Wall -g -std=c99 +LDLIBS = -ldl +realloc_test: realloc_test.c diff --git a/test/cpp/realloc/realloc_test.c b/test/cpp/realloc/realloc_test.c new file mode 100644 index 00000000..f9763adf --- /dev/null +++ b/test/cpp/realloc/realloc_test.c @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include + +int copies; +int non_copies; + +void *realloc(void *ptr, size_t size) { + static void *(*real_realloc)(void*, size_t) = NULL; + if (real_realloc == NULL) { + real_realloc = (void* (*) (void*, size_t)) dlsym(RTLD_NEXT, "realloc"); + } + + void *ret_ptr = (*real_realloc)(ptr, size); + + if (ret_ptr == ptr) { + non_copies++; + } else { + copies++; + } + + return ret_ptr; +} + + +struct TMemoryBuffer { + void* ptr; + int size; +}; + +int main(int argc, char *argv[]) { + int num_buffers; + int init_size; + int max_size; + int doublings; + int iterations; + + if (argc < 6 || + argc > 7 || + (num_buffers = atoi(argv[1])) == 0 || + (init_size = atoi(argv[2])) == 0 || + (max_size = atoi(argv[3])) == 0 || + init_size > max_size || + (iterations = atoi(argv[4])) == 0 || + (doublings = atoi(argv[5])) == 0 || + (argc == 7 && atoi(argv[6]) == 0)) { + fprintf(stderr, "usage: realloc_test [seed]\n"); + exit(EXIT_FAILURE); + } + + for ( int i = 0 ; i < argc ; i++ ) { + printf("%s ", argv[i]); + } + printf("\n"); + + if (argc == 7) { + srand(atoi(argv[6])); + } else { + srand(time(NULL)); + } + + struct TMemoryBuffer* buffers = calloc(num_buffers, sizeof(*buffers)); + if (buffers == NULL) abort(); + + for ( int i = 0 ; i < num_buffers ; i++ ) { + buffers[i].size = max_size; + } + + while (iterations --> 0) { + for ( int i = 0 ; i < doublings * num_buffers ; i++ ) { + struct TMemoryBuffer* buf = &buffers[rand() % num_buffers]; + buf->size *= 2; + if (buf->size <= max_size) { + buf->ptr = realloc(buf->ptr, buf->size); + } else { + free(buf->ptr); + buf->size = init_size; + buf->ptr = malloc(buf->size); + } + if (buf->ptr == NULL) abort(); + } + } + + printf("Non-copied %d/%d (%.2f%%)\n", non_copies, copies + non_copies, 100.0 * non_copies / (copies + non_copies)); + return 0; +} diff --git a/test/cpp/src/TestClient.cpp b/test/cpp/src/TestClient.cpp new file mode 100644 index 00000000..5ddfa060 --- /dev/null +++ b/test/cpp/src/TestClient.cpp @@ -0,0 +1,497 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include "ThriftTest.h" + +#define __STDC_FORMAT_MACROS +#include + +using namespace boost; +using namespace std; +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace thrift::test; + +//extern uint32_t g_socket_syscalls; + +// Current time, microseconds since the epoch +uint64_t now() +{ + long long ret; + struct timeval tv; + + gettimeofday(&tv, NULL); + ret = tv.tv_sec; + ret = ret*1000*1000 + tv.tv_usec; + return ret; +} + +int main(int argc, char** argv) { + string host = "localhost"; + int port = 9090; + int numTests = 1; + bool framed = false; + + for (int i = 0; i < argc; ++i) { + if (strcmp(argv[i], "-h") == 0) { + char* pch = strtok(argv[++i], ":"); + if (pch != NULL) { + host = string(pch); + } + pch = strtok(NULL, ":"); + if (pch != NULL) { + port = atoi(pch); + } + } else if (strcmp(argv[i], "-n") == 0) { + numTests = atoi(argv[++i]); + } else if (strcmp(argv[i], "-f") == 0) { + framed = true; + } + } + + + shared_ptr transport; + + shared_ptr socket(new TSocket(host, port)); + + if (framed) { + shared_ptr framedSocket(new TFramedTransport(socket)); + transport = framedSocket; + } else { + shared_ptr bufferedSocket(new TBufferedTransport(socket)); + transport = bufferedSocket; + } + + shared_ptr protocol(new TBinaryProtocol(transport)); + ThriftTestClient testClient(protocol); + + uint64_t time_min = 0; + uint64_t time_max = 0; + uint64_t time_tot = 0; + + int test = 0; + for (test = 0; test < numTests; ++test) { + + try { + transport->open(); + } catch (TTransportException& ttx) { + printf("Connect failed: %s\n", ttx.what()); + continue; + } + + /** + * CONNECT TEST + */ + printf("Test #%d, connect %s:%d\n", test+1, host.c_str(), port); + + uint64_t start = now(); + + /** + * VOID TEST + */ + try { + printf("testVoid()"); + testClient.testVoid(); + printf(" = void\n"); + } catch (TApplicationException tax) { + printf("%s\n", tax.what()); + } + + /** + * STRING TEST + */ + printf("testString(\"Test\")"); + string s; + testClient.testString(s, "Test"); + printf(" = \"%s\"\n", s.c_str()); + + /** + * BYTE TEST + */ + printf("testByte(1)"); + uint8_t u8 = testClient.testByte(1); + printf(" = %d\n", (int)u8); + + /** + * I32 TEST + */ + printf("testI32(-1)"); + int32_t i32 = testClient.testI32(-1); + printf(" = %d\n", i32); + + /** + * I64 TEST + */ + printf("testI64(-34359738368)"); + int64_t i64 = testClient.testI64(-34359738368LL); + printf(" = %"PRId64"\n", i64); + + /** + * DOUBLE TEST + */ + printf("testDouble(-5.2098523)"); + double dub = testClient.testDouble(-5.2098523); + printf(" = %lf\n", dub); + + /** + * STRUCT TEST + */ + printf("testStruct({\"Zero\", 1, -3, -5})"); + Xtruct out; + out.string_thing = "Zero"; + out.byte_thing = 1; + out.i32_thing = -3; + out.i64_thing = -5; + Xtruct in; + testClient.testStruct(in, out); + printf(" = {\"%s\", %d, %d, %"PRId64"}\n", + in.string_thing.c_str(), + (int)in.byte_thing, + in.i32_thing, + in.i64_thing); + + /** + * NESTED STRUCT TEST + */ + printf("testNest({1, {\"Zero\", 1, -3, -5}), 5}"); + Xtruct2 out2; + out2.byte_thing = 1; + out2.struct_thing = out; + out2.i32_thing = 5; + Xtruct2 in2; + testClient.testNest(in2, out2); + in = in2.struct_thing; + printf(" = {%d, {\"%s\", %d, %d, %"PRId64"}, %d}\n", + in2.byte_thing, + in.string_thing.c_str(), + (int)in.byte_thing, + in.i32_thing, + in.i64_thing, + in2.i32_thing); + + /** + * MAP TEST + */ + map mapout; + for (int32_t i = 0; i < 5; ++i) { + mapout.insert(make_pair(i, i-10)); + } + printf("testMap({"); + map::const_iterator m_iter; + bool first = true; + for (m_iter = mapout.begin(); m_iter != mapout.end(); ++m_iter) { + if (first) { + first = false; + } else { + printf(", "); + } + printf("%d => %d", m_iter->first, m_iter->second); + } + printf("})"); + map mapin; + testClient.testMap(mapin, mapout); + printf(" = {"); + first = true; + for (m_iter = mapin.begin(); m_iter != mapin.end(); ++m_iter) { + if (first) { + first = false; + } else { + printf(", "); + } + printf("%d => %d", m_iter->first, m_iter->second); + } + printf("}\n"); + + /** + * SET TEST + */ + set setout; + for (int32_t i = -2; i < 3; ++i) { + setout.insert(i); + } + printf("testSet({"); + set::const_iterator s_iter; + first = true; + for (s_iter = setout.begin(); s_iter != setout.end(); ++s_iter) { + if (first) { + first = false; + } else { + printf(", "); + } + printf("%d", *s_iter); + } + printf("})"); + set setin; + testClient.testSet(setin, setout); + printf(" = {"); + first = true; + for (s_iter = setin.begin(); s_iter != setin.end(); ++s_iter) { + if (first) { + first = false; + } else { + printf(", "); + } + printf("%d", *s_iter); + } + printf("}\n"); + + /** + * LIST TEST + */ + vector listout; + for (int32_t i = -2; i < 3; ++i) { + listout.push_back(i); + } + printf("testList({"); + vector::const_iterator l_iter; + first = true; + for (l_iter = listout.begin(); l_iter != listout.end(); ++l_iter) { + if (first) { + first = false; + } else { + printf(", "); + } + printf("%d", *l_iter); + } + printf("})"); + vector listin; + testClient.testList(listin, listout); + printf(" = {"); + first = true; + for (l_iter = listin.begin(); l_iter != listin.end(); ++l_iter) { + if (first) { + first = false; + } else { + printf(", "); + } + printf("%d", *l_iter); + } + printf("}\n"); + + /** + * ENUM TEST + */ + printf("testEnum(ONE)"); + Numberz ret = testClient.testEnum(ONE); + printf(" = %d\n", ret); + + printf("testEnum(TWO)"); + ret = testClient.testEnum(TWO); + printf(" = %d\n", ret); + + printf("testEnum(THREE)"); + ret = testClient.testEnum(THREE); + printf(" = %d\n", ret); + + printf("testEnum(FIVE)"); + ret = testClient.testEnum(FIVE); + printf(" = %d\n", ret); + + printf("testEnum(EIGHT)"); + ret = testClient.testEnum(EIGHT); + printf(" = %d\n", ret); + + /** + * TYPEDEF TEST + */ + printf("testTypedef(309858235082523)"); + UserId uid = testClient.testTypedef(309858235082523LL); + printf(" = %"PRId64"\n", uid); + + /** + * NESTED MAP TEST + */ + printf("testMapMap(1)"); + map > mm; + testClient.testMapMap(mm, 1); + printf(" = {"); + map >::const_iterator mi; + for (mi = mm.begin(); mi != mm.end(); ++mi) { + printf("%d => {", mi->first); + map::const_iterator mi2; + for (mi2 = mi->second.begin(); mi2 != mi->second.end(); ++mi2) { + printf("%d => %d, ", mi2->first, mi2->second); + } + printf("}, "); + } + printf("}\n"); + + /** + * INSANITY TEST + */ + Insanity insane; + insane.userMap.insert(make_pair(FIVE, 5000)); + Xtruct truck; + truck.string_thing = "Truck"; + truck.byte_thing = 8; + truck.i32_thing = 8; + truck.i64_thing = 8; + insane.xtructs.push_back(truck); + printf("testInsanity()"); + map > whoa; + testClient.testInsanity(whoa, insane); + printf(" = {"); + map >::const_iterator i_iter; + for (i_iter = whoa.begin(); i_iter != whoa.end(); ++i_iter) { + printf("%"PRId64" => {", i_iter->first); + map::const_iterator i2_iter; + for (i2_iter = i_iter->second.begin(); + i2_iter != i_iter->second.end(); + ++i2_iter) { + printf("%d => {", i2_iter->first); + map userMap = i2_iter->second.userMap; + map::const_iterator um; + printf("{"); + for (um = userMap.begin(); um != userMap.end(); ++um) { + printf("%d => %"PRId64", ", um->first, um->second); + } + printf("}, "); + + vector xtructs = i2_iter->second.xtructs; + vector::const_iterator x; + printf("{"); + for (x = xtructs.begin(); x != xtructs.end(); ++x) { + printf("{\"%s\", %d, %d, %"PRId64"}, ", + x->string_thing.c_str(), + (int)x->byte_thing, + x->i32_thing, + x->i64_thing); + } + printf("}"); + + printf("}, "); + } + printf("}, "); + } + printf("}\n"); + + /* test exception */ + + try { + printf("testClient.testException(\"Xception\") =>"); + testClient.testException("Xception"); + printf(" void\nFAILURE\n"); + + } catch(Xception& e) { + printf(" {%u, \"%s\"}\n", e.errorCode, e.message.c_str()); + } + + try { + printf("testClient.testException(\"success\") =>"); + testClient.testException("success"); + printf(" void\n"); + } catch(...) { + printf(" exception\nFAILURE\n"); + } + + /* test multi exception */ + + try { + printf("testClient.testMultiException(\"Xception\", \"test 1\") =>"); + Xtruct result; + testClient.testMultiException(result, "Xception", "test 1"); + printf(" result\nFAILURE\n"); + } catch(Xception& e) { + printf(" {%u, \"%s\"}\n", e.errorCode, e.message.c_str()); + } + + try { + printf("testClient.testMultiException(\"Xception2\", \"test 2\") =>"); + Xtruct result; + testClient.testMultiException(result, "Xception2", "test 2"); + printf(" result\nFAILURE\n"); + + } catch(Xception2& e) { + printf(" {%u, {\"%s\"}}\n", e.errorCode, e.struct_thing.string_thing.c_str()); + } + + try { + printf("testClient.testMultiException(\"success\", \"test 3\") =>"); + Xtruct result; + testClient.testMultiException(result, "success", "test 3"); + printf(" {{\"%s\"}}\n", result.string_thing.c_str()); + } catch(...) { + printf(" exception\nFAILURE\n"); + } + + /* test oneway void */ + { + printf("testClient.testOneway(3) =>"); + uint64_t startOneway = now(); + testClient.testOneway(3); + uint64_t elapsed = now() - startOneway; + if (elapsed > 200 * 1000) { // 0.2 seconds + printf(" FAILURE - took %.2f ms\n", (double)elapsed/1000.0); + } else { + printf(" success - took %.2f ms\n", (double)elapsed/1000.0); + } + } + + /** + * redo a simple test after the oneway to make sure we aren't "off by one" -- + * if the server treated oneway void like normal void, this next test will + * fail since it will get the void confirmation rather than the correct + * result. In this circumstance, the client will throw the exception: + * + * TApplicationException: Wrong method namea + */ + /** + * I32 TEST + */ + printf("re-test testI32(-1)"); + i32 = testClient.testI32(-1); + printf(" = %d\n", i32); + + + uint64_t stop = now(); + uint64_t tot = stop-start; + + printf("Total time: %"PRIu64" us\n", stop-start); + + time_tot += tot; + if (time_min == 0 || tot < time_min) { + time_min = tot; + } + if (tot > time_max) { + time_max = tot; + } + + transport->close(); + } + + // printf("\nSocket syscalls: %u", g_socket_syscalls); + printf("\nAll tests done.\n"); + + uint64_t time_avg = time_tot / numTests; + + printf("Min time: %"PRIu64" us\n", time_min); + printf("Max time: %"PRIu64" us\n", time_max); + printf("Avg time: %"PRIu64" us\n", time_avg); + + return 0; +} diff --git a/test/cpp/src/TestServer.cpp b/test/cpp/src/TestServer.cpp new file mode 100644 index 00000000..83454dca --- /dev/null +++ b/test/cpp/src/TestServer.cpp @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ThriftTest.h" + +#include +#include +#include + +#define __STDC_FORMAT_MACROS +#include + +using namespace std; +using namespace boost; + +using namespace apache::thrift; +using namespace apache::thrift::concurrency; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace apache::thrift::server; + +using namespace thrift::test; + +class TestHandler : public ThriftTestIf { + public: + TestHandler() {} + + void testVoid() { + printf("testVoid()\n"); + } + + void testString(string& out, const string &thing) { + printf("testString(\"%s\")\n", thing.c_str()); + out = thing; + } + + int8_t testByte(const int8_t thing) { + printf("testByte(%d)\n", (int)thing); + return thing; + } + + int32_t testI32(const int32_t thing) { + printf("testI32(%d)\n", thing); + return thing; + } + + int64_t testI64(const int64_t thing) { + printf("testI64(%"PRId64")\n", thing); + return thing; + } + + double testDouble(const double thing) { + printf("testDouble(%lf)\n", thing); + return thing; + } + + void testStruct(Xtruct& out, const Xtruct &thing) { + printf("testStruct({\"%s\", %d, %d, %"PRId64"})\n", thing.string_thing.c_str(), (int)thing.byte_thing, thing.i32_thing, thing.i64_thing); + out = thing; + } + + void testNest(Xtruct2& out, const Xtruct2& nest) { + const Xtruct &thing = nest.struct_thing; + printf("testNest({%d, {\"%s\", %d, %d, %"PRId64"}, %d})\n", (int)nest.byte_thing, thing.string_thing.c_str(), (int)thing.byte_thing, thing.i32_thing, thing.i64_thing, nest.i32_thing); + out = nest; + } + + void testMap(map &out, const map &thing) { + printf("testMap({"); + map::const_iterator m_iter; + bool first = true; + for (m_iter = thing.begin(); m_iter != thing.end(); ++m_iter) { + if (first) { + first = false; + } else { + printf(", "); + } + printf("%d => %d", m_iter->first, m_iter->second); + } + printf("})\n"); + out = thing; + } + + void testSet(set &out, const set &thing) { + printf("testSet({"); + set::const_iterator s_iter; + bool first = true; + for (s_iter = thing.begin(); s_iter != thing.end(); ++s_iter) { + if (first) { + first = false; + } else { + printf(", "); + } + printf("%d", *s_iter); + } + printf("})\n"); + out = thing; + } + + void testList(vector &out, const vector &thing) { + printf("testList({"); + vector::const_iterator l_iter; + bool first = true; + for (l_iter = thing.begin(); l_iter != thing.end(); ++l_iter) { + if (first) { + first = false; + } else { + printf(", "); + } + printf("%d", *l_iter); + } + printf("})\n"); + out = thing; + } + + Numberz testEnum(const Numberz thing) { + printf("testEnum(%d)\n", thing); + return thing; + } + + UserId testTypedef(const UserId thing) { + printf("testTypedef(%"PRId64")\n", thing); + return thing; + } + + void testMapMap(map > &mapmap, const int32_t hello) { + printf("testMapMap(%d)\n", hello); + + map pos; + map neg; + for (int i = 1; i < 5; i++) { + pos.insert(make_pair(i,i)); + neg.insert(make_pair(-i,-i)); + } + + mapmap.insert(make_pair(4, pos)); + mapmap.insert(make_pair(-4, neg)); + + } + + void testInsanity(map > &insane, const Insanity &argument) { + printf("testInsanity()\n"); + + Xtruct hello; + hello.string_thing = "Hello2"; + hello.byte_thing = 2; + hello.i32_thing = 2; + hello.i64_thing = 2; + + Xtruct goodbye; + goodbye.string_thing = "Goodbye4"; + goodbye.byte_thing = 4; + goodbye.i32_thing = 4; + goodbye.i64_thing = 4; + + Insanity crazy; + crazy.userMap.insert(make_pair(EIGHT, 8)); + crazy.xtructs.push_back(goodbye); + + Insanity looney; + crazy.userMap.insert(make_pair(FIVE, 5)); + crazy.xtructs.push_back(hello); + + map first_map; + map second_map; + + first_map.insert(make_pair(TWO, crazy)); + first_map.insert(make_pair(THREE, crazy)); + + second_map.insert(make_pair(SIX, looney)); + + insane.insert(make_pair(1, first_map)); + insane.insert(make_pair(2, second_map)); + + printf("return"); + printf(" = {"); + map >::const_iterator i_iter; + for (i_iter = insane.begin(); i_iter != insane.end(); ++i_iter) { + printf("%"PRId64" => {", i_iter->first); + map::const_iterator i2_iter; + for (i2_iter = i_iter->second.begin(); + i2_iter != i_iter->second.end(); + ++i2_iter) { + printf("%d => {", i2_iter->first); + map userMap = i2_iter->second.userMap; + map::const_iterator um; + printf("{"); + for (um = userMap.begin(); um != userMap.end(); ++um) { + printf("%d => %"PRId64", ", um->first, um->second); + } + printf("}, "); + + vector xtructs = i2_iter->second.xtructs; + vector::const_iterator x; + printf("{"); + for (x = xtructs.begin(); x != xtructs.end(); ++x) { + printf("{\"%s\", %d, %d, %"PRId64"}, ", x->string_thing.c_str(), (int)x->byte_thing, x->i32_thing, x->i64_thing); + } + printf("}"); + + printf("}, "); + } + printf("}, "); + } + printf("}\n"); + + + } + + void testMulti(Xtruct &hello, const int8_t arg0, const int32_t arg1, const int64_t arg2, const std::map &arg3, const Numberz arg4, const UserId arg5) { + printf("testMulti()\n"); + + hello.string_thing = "Hello2"; + hello.byte_thing = arg0; + hello.i32_thing = arg1; + hello.i64_thing = (int64_t)arg2; + } + + void testException(const std::string &arg) + throw(Xception, apache::thrift::TException) + { + printf("testException(%s)\n", arg.c_str()); + if (arg.compare("Xception") == 0) { + Xception e; + e.errorCode = 1001; + e.message = arg; + throw e; + } else if (arg.compare("ApplicationException") == 0) { + apache::thrift::TException e; + throw e; + } else { + Xtruct result; + result.string_thing = arg; + return; + } + } + + void testMultiException(Xtruct &result, const std::string &arg0, const std::string &arg1) throw(Xception, Xception2) { + + printf("testMultiException(%s, %s)\n", arg0.c_str(), arg1.c_str()); + + if (arg0.compare("Xception") == 0) { + Xception e; + e.errorCode = 1001; + e.message = "This is an Xception"; + throw e; + } else if (arg0.compare("Xception2") == 0) { + Xception2 e; + e.errorCode = 2002; + e.struct_thing.string_thing = "This is an Xception2"; + throw e; + } else { + result.string_thing = arg1; + return; + } + } + + void testOneway(int sleepFor) { + printf("testOneway(%d): Sleeping...\n", sleepFor); + sleep(sleepFor); + printf("testOneway(%d): done sleeping!\n", sleepFor); + } +}; + +int main(int argc, char **argv) { + + int port = 9090; + string serverType = "simple"; + string protocolType = "binary"; + size_t workerCount = 4; + + ostringstream usage; + + usage << + argv[0] << " [--port=] [--server-type=] [--protocol-type=] [--workers=]" << endl << + + "\t\tserver-type\t\ttype of server, \"simple\", \"thread-pool\", \"threaded\", or \"nonblocking\". Default is " << serverType << endl << + + "\t\tprotocol-type\t\ttype of protocol, \"binary\", \"ascii\", or \"xml\". Default is " << protocolType << endl << + + "\t\tworkers\t\tNumber of thread pools workers. Only valid for thread-pool server type. Default is " << workerCount << endl; + + map args; + + for (int ix = 1; ix < argc; ix++) { + string arg(argv[ix]); + if (arg.compare(0,2, "--") == 0) { + size_t end = arg.find_first_of("=", 2); + if (end != string::npos) { + args[string(arg, 2, end - 2)] = string(arg, end + 1); + } else { + args[string(arg, 2)] = "true"; + } + } else { + throw invalid_argument("Unexcepted command line token: "+arg); + } + } + + try { + + if (!args["port"].empty()) { + port = atoi(args["port"].c_str()); + } + + if (!args["server-type"].empty()) { + serverType = args["server-type"]; + if (serverType == "simple") { + } else if (serverType == "thread-pool") { + } else if (serverType == "threaded") { + } else if (serverType == "nonblocking") { + } else { + throw invalid_argument("Unknown server type "+serverType); + } + } + + if (!args["protocol-type"].empty()) { + protocolType = args["protocol-type"]; + if (protocolType == "binary") { + } else if (protocolType == "ascii") { + throw invalid_argument("ASCII protocol not supported"); + } else if (protocolType == "xml") { + throw invalid_argument("XML protocol not supported"); + } else { + throw invalid_argument("Unknown protocol type "+protocolType); + } + } + + if (!args["workers"].empty()) { + workerCount = atoi(args["workers"].c_str()); + } + } catch (exception& e) { + cerr << e.what() << endl; + cerr << usage; + } + + // Dispatcher + shared_ptr protocolFactory(new TBinaryProtocolFactory()); + + shared_ptr testHandler(new TestHandler()); + + shared_ptr testProcessor(new ThriftTestProcessor(testHandler)); + + // Transport + shared_ptr serverSocket(new TServerSocket(port)); + + // Factory + shared_ptr transportFactory(new TBufferedTransportFactory()); + + if (serverType == "simple") { + + // Server + TSimpleServer simpleServer(testProcessor, + serverSocket, + transportFactory, + protocolFactory); + + printf("Starting the server on port %d...\n", port); + simpleServer.serve(); + + } else if (serverType == "thread-pool") { + + shared_ptr threadManager = + ThreadManager::newSimpleThreadManager(workerCount); + + shared_ptr threadFactory = + shared_ptr(new PosixThreadFactory()); + + threadManager->threadFactory(threadFactory); + + threadManager->start(); + + TThreadPoolServer threadPoolServer(testProcessor, + serverSocket, + transportFactory, + protocolFactory, + threadManager); + + printf("Starting the server on port %d...\n", port); + threadPoolServer.serve(); + + } else if (serverType == "threaded") { + + TThreadedServer threadedServer(testProcessor, + serverSocket, + transportFactory, + protocolFactory); + + printf("Starting the server on port %d...\n", port); + threadedServer.serve(); + + } else if (serverType == "nonblocking") { + TNonblockingServer nonblockingServer(testProcessor, port); + printf("Starting the nonblocking server on port %d...\n", port); + nonblockingServer.serve(); + } + + printf("done.\n"); + return 0; +} diff --git a/test/cpp/src/main.cpp b/test/cpp/src/main.cpp new file mode 100644 index 00000000..46ee950d --- /dev/null +++ b/test/cpp/src/main.cpp @@ -0,0 +1,509 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Service.h" + +#include +#include +#include +#include + +#include +#include +using __gnu_cxx::hash_map; +using __gnu_cxx::hash; + +using namespace std; +using namespace boost; + +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace apache::thrift::server; +using namespace apache::thrift::concurrency; + +using namespace test::stress; + +struct eqstr { + bool operator()(const char* s1, const char* s2) const { + return strcmp(s1, s2) == 0; + } +}; + +struct ltstr { + bool operator()(const char* s1, const char* s2) const { + return strcmp(s1, s2) < 0; + } +}; + + +// typedef hash_map, eqstr> count_map; +typedef map count_map; + +class Server : public ServiceIf { + public: + Server() {} + + void count(const char* method) { + Guard m(lock_); + int ct = counts_[method]; + counts_[method] = ++ct; + } + + void echoVoid() { + count("echoVoid"); + return; + } + + count_map getCount() { + Guard m(lock_); + return counts_; + } + + int8_t echoByte(const int8_t arg) {return arg;} + int32_t echoI32(const int32_t arg) {return arg;} + int64_t echoI64(const int64_t arg) {return arg;} + void echoString(string& out, const string &arg) { + if (arg != "hello") { + T_ERROR_ABORT("WRONG STRING!!!!"); + } + out = arg; + } + void echoList(vector &out, const vector &arg) { out = arg; } + void echoSet(set &out, const set &arg) { out = arg; } + void echoMap(map &out, const map &arg) { out = arg; } + +private: + count_map counts_; + Mutex lock_; + +}; + +class ClientThread: public Runnable { +public: + + ClientThread(shared_ptrtransport, shared_ptr client, Monitor& monitor, size_t& workerCount, size_t loopCount, TType loopType) : + _transport(transport), + _client(client), + _monitor(monitor), + _workerCount(workerCount), + _loopCount(loopCount), + _loopType(loopType) + {} + + void run() { + + // Wait for all worker threads to start + + {Synchronized s(_monitor); + while(_workerCount == 0) { + _monitor.wait(); + } + } + + _startTime = Util::currentTime(); + + _transport->open(); + + switch(_loopType) { + case T_VOID: loopEchoVoid(); break; + case T_BYTE: loopEchoByte(); break; + case T_I32: loopEchoI32(); break; + case T_I64: loopEchoI64(); break; + case T_STRING: loopEchoString(); break; + default: cerr << "Unexpected loop type" << _loopType << endl; break; + } + + _endTime = Util::currentTime(); + + _transport->close(); + + _done = true; + + {Synchronized s(_monitor); + + _workerCount--; + + if (_workerCount == 0) { + + _monitor.notify(); + } + } + } + + void loopEchoVoid() { + for (size_t ix = 0; ix < _loopCount; ix++) { + _client->echoVoid(); + } + } + + void loopEchoByte() { + for (size_t ix = 0; ix < _loopCount; ix++) { + int8_t arg = 1; + int8_t result; + result =_client->echoByte(arg); + assert(result == arg); + } + } + + void loopEchoI32() { + for (size_t ix = 0; ix < _loopCount; ix++) { + int32_t arg = 1; + int32_t result; + result =_client->echoI32(arg); + assert(result == arg); + } + } + + void loopEchoI64() { + for (size_t ix = 0; ix < _loopCount; ix++) { + int64_t arg = 1; + int64_t result; + result =_client->echoI64(arg); + assert(result == arg); + } + } + + void loopEchoString() { + for (size_t ix = 0; ix < _loopCount; ix++) { + string arg = "hello"; + string result; + _client->echoString(result, arg); + assert(result == arg); + } + } + + shared_ptr _transport; + shared_ptr _client; + Monitor& _monitor; + size_t& _workerCount; + size_t _loopCount; + TType _loopType; + long long _startTime; + long long _endTime; + bool _done; + Monitor _sleep; +}; + + +int main(int argc, char **argv) { + + int port = 9091; + string serverType = "thread-pool"; + string protocolType = "binary"; + size_t workerCount = 4; + size_t clientCount = 20; + size_t loopCount = 50000; + TType loopType = T_VOID; + string callName = "echoVoid"; + bool runServer = true; + bool logRequests = false; + string requestLogPath = "./requestlog.tlog"; + bool replayRequests = false; + + ostringstream usage; + + usage << + argv[0] << " [--port=] [--server] [--server-type=] [--protocol-type=] [--workers=] [--clients=] [--loop=]" << endl << + "\tclients Number of client threads to create - 0 implies no clients, i.e. server only. Default is " << clientCount << endl << + "\thelp Prints this help text." << endl << + "\tcall Service method to call. Default is " << callName << endl << + "\tloop The number of remote thrift calls each client makes. Default is " << loopCount << endl << + "\tport The port the server and clients should bind to for thrift network connections. Default is " << port << endl << + "\tserver Run the Thrift server in this process. Default is " << runServer << endl << + "\tserver-type Type of server, \"simple\" or \"thread-pool\". Default is " << serverType << endl << + "\tprotocol-type Type of protocol, \"binary\", \"ascii\", or \"xml\". Default is " << protocolType << endl << + "\tlog-request Log all request to ./requestlog.tlog. Default is " << logRequests << endl << + "\treplay-request Replay requests from log file (./requestlog.tlog) Default is " << replayRequests << endl << + "\tworkers Number of thread pools workers. Only valid for thread-pool server type. Default is " << workerCount << endl; + + + map args; + + for (int ix = 1; ix < argc; ix++) { + + string arg(argv[ix]); + + if (arg.compare(0,2, "--") == 0) { + + size_t end = arg.find_first_of("=", 2); + + string key = string(arg, 2, end - 2); + + if (end != string::npos) { + args[key] = string(arg, end + 1); + } else { + args[key] = "true"; + } + } else { + throw invalid_argument("Unexcepted command line token: "+arg); + } + } + + try { + + if (!args["clients"].empty()) { + clientCount = atoi(args["clients"].c_str()); + } + + if (!args["help"].empty()) { + cerr << usage.str(); + return 0; + } + + if (!args["loop"].empty()) { + loopCount = atoi(args["loop"].c_str()); + } + + if (!args["call"].empty()) { + callName = args["call"]; + } + + if (!args["port"].empty()) { + port = atoi(args["port"].c_str()); + } + + if (!args["server"].empty()) { + runServer = args["server"] == "true"; + } + + if (!args["log-request"].empty()) { + logRequests = args["log-request"] == "true"; + } + + if (!args["replay-request"].empty()) { + replayRequests = args["replay-request"] == "true"; + } + + if (!args["server-type"].empty()) { + serverType = args["server-type"]; + + if (serverType == "simple") { + + } else if (serverType == "thread-pool") { + + } else if (serverType == "threaded") { + + } else { + + throw invalid_argument("Unknown server type "+serverType); + } + } + + if (!args["workers"].empty()) { + workerCount = atoi(args["workers"].c_str()); + } + + } catch(exception& e) { + cerr << e.what() << endl; + cerr << usage; + } + + shared_ptr threadFactory = shared_ptr(new PosixThreadFactory()); + + // Dispatcher + shared_ptr serviceHandler(new Server()); + + if (replayRequests) { + shared_ptr serviceHandler(new Server()); + shared_ptr serviceProcessor(new ServiceProcessor(serviceHandler)); + + // Transports + shared_ptr fileTransport(new TFileTransport(requestLogPath)); + fileTransport->setChunkSize(2 * 1024 * 1024); + fileTransport->setMaxEventSize(1024 * 16); + fileTransport->seekToEnd(); + + // Protocol Factory + shared_ptr protocolFactory(new TBinaryProtocolFactory()); + + TFileProcessor fileProcessor(serviceProcessor, + protocolFactory, + fileTransport); + + fileProcessor.process(0, true); + exit(0); + } + + + if (runServer) { + + shared_ptr serviceProcessor(new ServiceProcessor(serviceHandler)); + + // Transport + shared_ptr serverSocket(new TServerSocket(port)); + + // Transport Factory + shared_ptr transportFactory(new TBufferedTransportFactory()); + + // Protocol Factory + shared_ptr protocolFactory(new TBinaryProtocolFactory()); + + if (logRequests) { + // initialize the log file + shared_ptr fileTransport(new TFileTransport(requestLogPath)); + fileTransport->setChunkSize(2 * 1024 * 1024); + fileTransport->setMaxEventSize(1024 * 16); + + transportFactory = + shared_ptr(new TPipedTransportFactory(fileTransport)); + } + + shared_ptr serverThread; + + if (serverType == "simple") { + + serverThread = threadFactory->newThread(shared_ptr(new TSimpleServer(serviceProcessor, serverSocket, transportFactory, protocolFactory))); + + } else if (serverType == "threaded") { + + serverThread = threadFactory->newThread(shared_ptr(new TThreadedServer(serviceProcessor, serverSocket, transportFactory, protocolFactory))); + + } else if (serverType == "thread-pool") { + + shared_ptr threadManager = ThreadManager::newSimpleThreadManager(workerCount); + + threadManager->threadFactory(threadFactory); + threadManager->start(); + serverThread = threadFactory->newThread(shared_ptr(new TThreadPoolServer(serviceProcessor, serverSocket, transportFactory, protocolFactory, threadManager))); + } + + cerr << "Starting the server on port " << port << endl; + + serverThread->start(); + + // If we aren't running clients, just wait forever for external clients + + if (clientCount == 0) { + serverThread->join(); + } + } + + if (clientCount > 0) { + + Monitor monitor; + + size_t threadCount = 0; + + set > clientThreads; + + if (callName == "echoVoid") { loopType = T_VOID;} + else if (callName == "echoByte") { loopType = T_BYTE;} + else if (callName == "echoI32") { loopType = T_I32;} + else if (callName == "echoI64") { loopType = T_I64;} + else if (callName == "echoString") { loopType = T_STRING;} + else {throw invalid_argument("Unknown service call "+callName);} + + for (size_t ix = 0; ix < clientCount; ix++) { + + shared_ptr socket(new TSocket("127.0.01", port)); + shared_ptr bufferedSocket(new TBufferedTransport(socket, 2048)); + shared_ptr protocol(new TBinaryProtocol(bufferedSocket)); + shared_ptr serviceClient(new ServiceClient(protocol)); + + clientThreads.insert(threadFactory->newThread(shared_ptr(new ClientThread(socket, serviceClient, monitor, threadCount, loopCount, loopType)))); + } + + for (std::set >::const_iterator thread = clientThreads.begin(); thread != clientThreads.end(); thread++) { + (*thread)->start(); + } + + long long time00; + long long time01; + + {Synchronized s(monitor); + threadCount = clientCount; + + cerr << "Launch "<< clientCount << " client threads" << endl; + + time00 = Util::currentTime(); + + monitor.notifyAll(); + + while(threadCount > 0) { + monitor.wait(); + } + + time01 = Util::currentTime(); + } + + long long firstTime = 9223372036854775807LL; + long long lastTime = 0; + + double averageTime = 0; + long long minTime = 9223372036854775807LL; + long long maxTime = 0; + + for (set >::iterator ix = clientThreads.begin(); ix != clientThreads.end(); ix++) { + + shared_ptr client = dynamic_pointer_cast((*ix)->runnable()); + + long long delta = client->_endTime - client->_startTime; + + assert(delta > 0); + + if (client->_startTime < firstTime) { + firstTime = client->_startTime; + } + + if (client->_endTime > lastTime) { + lastTime = client->_endTime; + } + + if (delta < minTime) { + minTime = delta; + } + + if (delta > maxTime) { + maxTime = delta; + } + + averageTime+= delta; + } + + averageTime /= clientCount; + + + cout << "workers :" << workerCount << ", client : " << clientCount << ", loops : " << loopCount << ", rate : " << (clientCount * loopCount * 1000) / ((double)(time01 - time00)) << endl; + + count_map count = serviceHandler->getCount(); + count_map::iterator iter; + for (iter = count.begin(); iter != count.end(); ++iter) { + printf("%s => %d\n", iter->first, iter->second); + } + cerr << "done." << endl; + } + + return 0; +} diff --git a/test/cpp/src/nb-main.cpp b/test/cpp/src/nb-main.cpp new file mode 100644 index 00000000..8c74a815 --- /dev/null +++ b/test/cpp/src/nb-main.cpp @@ -0,0 +1,502 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Service.h" + +#include +#include + +#include +#include +#include +#include + +#include +#include +using __gnu_cxx::hash_map; +using __gnu_cxx::hash; + +using namespace std; +using namespace boost; + +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace apache::thrift::server; +using namespace apache::thrift::concurrency; + +using namespace test::stress; + +struct eqstr { + bool operator()(const char* s1, const char* s2) const { + return strcmp(s1, s2) == 0; + } +}; + +struct ltstr { + bool operator()(const char* s1, const char* s2) const { + return strcmp(s1, s2) < 0; + } +}; + + +// typedef hash_map, eqstr> count_map; +typedef map count_map; + +class Server : public ServiceIf { + public: + Server() {} + + void count(const char* method) { + Guard m(lock_); + int ct = counts_[method]; + counts_[method] = ++ct; + } + + void echoVoid() { + count("echoVoid"); + // Sleep to simulate work + usleep(5000); + return; + } + + count_map getCount() { + Guard m(lock_); + return counts_; + } + + int8_t echoByte(const int8_t arg) {return arg;} + int32_t echoI32(const int32_t arg) {return arg;} + int64_t echoI64(const int64_t arg) {return arg;} + void echoString(string& out, const string &arg) { + if (arg != "hello") { + T_ERROR_ABORT("WRONG STRING!!!!"); + } + out = arg; + } + void echoList(vector &out, const vector &arg) { out = arg; } + void echoSet(set &out, const set &arg) { out = arg; } + void echoMap(map &out, const map &arg) { out = arg; } + +private: + count_map counts_; + Mutex lock_; + +}; + +class ClientThread: public Runnable { +public: + + ClientThread(shared_ptrtransport, shared_ptr client, Monitor& monitor, size_t& workerCount, size_t loopCount, TType loopType) : + _transport(transport), + _client(client), + _monitor(monitor), + _workerCount(workerCount), + _loopCount(loopCount), + _loopType(loopType) + {} + + void run() { + + // Wait for all worker threads to start + + {Synchronized s(_monitor); + while(_workerCount == 0) { + _monitor.wait(); + } + } + + _startTime = Util::currentTime(); + + _transport->open(); + + switch(_loopType) { + case T_VOID: loopEchoVoid(); break; + case T_BYTE: loopEchoByte(); break; + case T_I32: loopEchoI32(); break; + case T_I64: loopEchoI64(); break; + case T_STRING: loopEchoString(); break; + default: cerr << "Unexpected loop type" << _loopType << endl; break; + } + + _endTime = Util::currentTime(); + + _transport->close(); + + _done = true; + + {Synchronized s(_monitor); + + _workerCount--; + + if (_workerCount == 0) { + + _monitor.notify(); + } + } + } + + void loopEchoVoid() { + for (size_t ix = 0; ix < _loopCount; ix++) { + _client->echoVoid(); + } + } + + void loopEchoByte() { + for (size_t ix = 0; ix < _loopCount; ix++) { + int8_t arg = 1; + int8_t result; + result =_client->echoByte(arg); + assert(result == arg); + } + } + + void loopEchoI32() { + for (size_t ix = 0; ix < _loopCount; ix++) { + int32_t arg = 1; + int32_t result; + result =_client->echoI32(arg); + assert(result == arg); + } + } + + void loopEchoI64() { + for (size_t ix = 0; ix < _loopCount; ix++) { + int64_t arg = 1; + int64_t result; + result =_client->echoI64(arg); + assert(result == arg); + } + } + + void loopEchoString() { + for (size_t ix = 0; ix < _loopCount; ix++) { + string arg = "hello"; + string result; + _client->echoString(result, arg); + assert(result == arg); + } + } + + shared_ptr _transport; + shared_ptr _client; + Monitor& _monitor; + size_t& _workerCount; + size_t _loopCount; + TType _loopType; + long long _startTime; + long long _endTime; + bool _done; + Monitor _sleep; +}; + + +int main(int argc, char **argv) { + + int port = 9091; + string serverType = "simple"; + string protocolType = "binary"; + size_t workerCount = 4; + size_t clientCount = 20; + size_t loopCount = 50000; + TType loopType = T_VOID; + string callName = "echoVoid"; + bool runServer = true; + bool logRequests = false; + string requestLogPath = "./requestlog.tlog"; + bool replayRequests = false; + + ostringstream usage; + + usage << + argv[0] << " [--port=] [--server] [--server-type=] [--protocol-type=] [--workers=] [--clients=] [--loop=]" << endl << + "\tclients Number of client threads to create - 0 implies no clients, i.e. server only. Default is " << clientCount << endl << + "\thelp Prints this help text." << endl << + "\tcall Service method to call. Default is " << callName << endl << + "\tloop The number of remote thrift calls each client makes. Default is " << loopCount << endl << + "\tport The port the server and clients should bind to for thrift network connections. Default is " << port << endl << + "\tserver Run the Thrift server in this process. Default is " << runServer << endl << + "\tserver-type Type of server, \"simple\" or \"thread-pool\". Default is " << serverType << endl << + "\tprotocol-type Type of protocol, \"binary\", \"ascii\", or \"xml\". Default is " << protocolType << endl << + "\tlog-request Log all request to ./requestlog.tlog. Default is " << logRequests << endl << + "\treplay-request Replay requests from log file (./requestlog.tlog) Default is " << replayRequests << endl << + "\tworkers Number of thread pools workers. Only valid for thread-pool server type. Default is " << workerCount << endl; + + + map args; + + for (int ix = 1; ix < argc; ix++) { + + string arg(argv[ix]); + + if (arg.compare(0,2, "--") == 0) { + + size_t end = arg.find_first_of("=", 2); + + string key = string(arg, 2, end - 2); + + if (end != string::npos) { + args[key] = string(arg, end + 1); + } else { + args[key] = "true"; + } + } else { + throw invalid_argument("Unexcepted command line token: "+arg); + } + } + + try { + + if (!args["clients"].empty()) { + clientCount = atoi(args["clients"].c_str()); + } + + if (!args["help"].empty()) { + cerr << usage.str(); + return 0; + } + + if (!args["loop"].empty()) { + loopCount = atoi(args["loop"].c_str()); + } + + if (!args["call"].empty()) { + callName = args["call"]; + } + + if (!args["port"].empty()) { + port = atoi(args["port"].c_str()); + } + + if (!args["server"].empty()) { + runServer = args["server"] == "true"; + } + + if (!args["log-request"].empty()) { + logRequests = args["log-request"] == "true"; + } + + if (!args["replay-request"].empty()) { + replayRequests = args["replay-request"] == "true"; + } + + if (!args["server-type"].empty()) { + serverType = args["server-type"]; + } + + if (!args["workers"].empty()) { + workerCount = atoi(args["workers"].c_str()); + } + + } catch(exception& e) { + cerr << e.what() << endl; + cerr << usage; + } + + shared_ptr threadFactory = shared_ptr(new PosixThreadFactory()); + + // Dispatcher + shared_ptr serviceHandler(new Server()); + + if (replayRequests) { + shared_ptr serviceHandler(new Server()); + shared_ptr serviceProcessor(new ServiceProcessor(serviceHandler)); + + // Transports + shared_ptr fileTransport(new TFileTransport(requestLogPath)); + fileTransport->setChunkSize(2 * 1024 * 1024); + fileTransport->setMaxEventSize(1024 * 16); + fileTransport->seekToEnd(); + + // Protocol Factory + shared_ptr protocolFactory(new TBinaryProtocolFactory()); + + TFileProcessor fileProcessor(serviceProcessor, + protocolFactory, + fileTransport); + + fileProcessor.process(0, true); + exit(0); + } + + + if (runServer) { + + shared_ptr serviceProcessor(new ServiceProcessor(serviceHandler)); + + // Protocol Factory + shared_ptr protocolFactory(new TBinaryProtocolFactory()); + + // Transport Factory + shared_ptr transportFactory; + + if (logRequests) { + // initialize the log file + shared_ptr fileTransport(new TFileTransport(requestLogPath)); + fileTransport->setChunkSize(2 * 1024 * 1024); + fileTransport->setMaxEventSize(1024 * 16); + + transportFactory = + shared_ptr(new TPipedTransportFactory(fileTransport)); + } + + shared_ptr serverThread; + shared_ptr serverThread2; + + if (serverType == "simple") { + + serverThread = threadFactory->newThread(shared_ptr(new TNonblockingServer(serviceProcessor, protocolFactory, port))); + serverThread2 = threadFactory->newThread(shared_ptr(new TNonblockingServer(serviceProcessor, protocolFactory, port+1))); + + } else if (serverType == "thread-pool") { + + shared_ptr threadManager = ThreadManager::newSimpleThreadManager(workerCount); + + threadManager->threadFactory(threadFactory); + threadManager->start(); + serverThread = threadFactory->newThread(shared_ptr(new TNonblockingServer(serviceProcessor, protocolFactory, port, threadManager))); + serverThread2 = threadFactory->newThread(shared_ptr(new TNonblockingServer(serviceProcessor, protocolFactory, port+1, threadManager))); + } + + cerr << "Starting the server on port " << port << " and " << (port + 1) << endl; + serverThread->start(); + serverThread2->start(); + + // If we aren't running clients, just wait forever for external clients + + if (clientCount == 0) { + serverThread->join(); + serverThread2->join(); + } + } + sleep(1); + + if (clientCount > 0) { + + Monitor monitor; + + size_t threadCount = 0; + + set > clientThreads; + + if (callName == "echoVoid") { loopType = T_VOID;} + else if (callName == "echoByte") { loopType = T_BYTE;} + else if (callName == "echoI32") { loopType = T_I32;} + else if (callName == "echoI64") { loopType = T_I64;} + else if (callName == "echoString") { loopType = T_STRING;} + else {throw invalid_argument("Unknown service call "+callName);} + + for (size_t ix = 0; ix < clientCount; ix++) { + + shared_ptr socket(new TSocket("127.0.0.1", port + (ix % 2))); + shared_ptr framedSocket(new TFramedTransport(socket)); + shared_ptr protocol(new TBinaryProtocol(framedSocket)); + shared_ptr serviceClient(new ServiceClient(protocol)); + + clientThreads.insert(threadFactory->newThread(shared_ptr(new ClientThread(socket, serviceClient, monitor, threadCount, loopCount, loopType)))); + } + + for (std::set >::const_iterator thread = clientThreads.begin(); thread != clientThreads.end(); thread++) { + (*thread)->start(); + } + + long long time00; + long long time01; + + {Synchronized s(monitor); + threadCount = clientCount; + + cerr << "Launch "<< clientCount << " client threads" << endl; + + time00 = Util::currentTime(); + + monitor.notifyAll(); + + while(threadCount > 0) { + monitor.wait(); + } + + time01 = Util::currentTime(); + } + + long long firstTime = 9223372036854775807LL; + long long lastTime = 0; + + double averageTime = 0; + long long minTime = 9223372036854775807LL; + long long maxTime = 0; + + for (set >::iterator ix = clientThreads.begin(); ix != clientThreads.end(); ix++) { + + shared_ptr client = dynamic_pointer_cast((*ix)->runnable()); + + long long delta = client->_endTime - client->_startTime; + + assert(delta > 0); + + if (client->_startTime < firstTime) { + firstTime = client->_startTime; + } + + if (client->_endTime > lastTime) { + lastTime = client->_endTime; + } + + if (delta < minTime) { + minTime = delta; + } + + if (delta > maxTime) { + maxTime = delta; + } + + averageTime+= delta; + } + + averageTime /= clientCount; + + + cout << "workers :" << workerCount << ", client : " << clientCount << ", loops : " << loopCount << ", rate : " << (clientCount * loopCount * 1000) / ((double)(time01 - time00)) << endl; + + count_map count = serviceHandler->getCount(); + count_map::iterator iter; + for (iter = count.begin(); iter != count.end(); ++iter) { + printf("%s => %d\n", iter->first, iter->second); + } + cerr << "done." << endl; + } + + return 0; +} diff --git a/test/csharp/CSharpClient.cs b/test/csharp/CSharpClient.cs new file mode 100644 index 00000000..641d5c9b --- /dev/null +++ b/test/csharp/CSharpClient.cs @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using Thrift; +using Thrift.Protocol; +using Thrift.Server; +using Thrift.Transport; + + +namespace CSharpTutorial +{ + public class CSharpClient + { + public static void Main() + { + try + { + TTransport transport = new TSocket("localhost", 9090); + TProtocol protocol = new TBinaryProtocol(transport); + Calculator.Client client = new Calculator.Client(protocol); + + transport.Open(); + + client.ping(); + Console.WriteLine("ping()"); + + int sum = client.add(1, 1); + Console.WriteLine("1+1={0}", sum); + + Work work = new Work(); + + work.op = Operation.DIVIDE; + work.num1 = 1; + work.num2 = 0; + try + { + int quotient = client.calculate(1, work); + Console.WriteLine("Whoa we can divide by 0"); + } + catch (InvalidOperation io) + { + Console.WriteLine("Invalid operation: " + io.why); + } + + work.op = Operation.SUBTRACT; + work.num1 = 15; + work.num2 = 10; + try + { + int diff = client.calculate(1, work); + Console.WriteLine("15-10={0}", diff); + } + catch (InvalidOperation io) + { + Console.WriteLine("Invalid operation: " + io.why); + } + + SharedStruct log = client.getStruct(1); + Console.WriteLine("Check log: {0}", log.value); + + transport.Close(); + } + catch (TApplicationException x) + { + Console.WriteLine(x.StackTrace); + } + + } + } +} diff --git a/test/csharp/CSharpServer.cs b/test/csharp/CSharpServer.cs new file mode 100644 index 00000000..f9ab8fd2 --- /dev/null +++ b/test/csharp/CSharpServer.cs @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using Thrift.Server; +using Thrift.Transport; + +namespace CSharpTutorial +{ + public class CalculatorHandler : Calculator.Iface + { + Dictionary log; + + public CalculatorHandler() + { + log = new Dictionary(); + } + + public void ping() + { + Console.WriteLine("ping()"); + } + + public int add(int n1, int n2) + { + Console.WriteLine("add({0},{1})", n1, n2); + return n1 + n2; + } + + public int calculate(int logid, Work work) + { + Console.WriteLine("calculate({0}, [{1},{2},{3}])", logid, work.op, work.num1, work.num2); + int val = 0; + switch (work.op) + { + case Operation.ADD: + val = work.num1 + work.num2; + break; + + case Operation.SUBTRACT: + val = work.num1 - work.num2; + break; + + case Operation.MULTIPLY: + val = work.num1 * work.num2; + break; + + case Operation.DIVIDE: + if (work.num2 == 0) + { + InvalidOperation io = new InvalidOperation(); + io.what = (int)work.op; + io.why = "Cannot divide by 0"; + throw io; + } + val = work.num1 / work.num2; + break; + + default: + { + InvalidOperation io = new InvalidOperation(); + io.what = (int)work.op; + io.why = "Unknown operation"; + throw io; + } + } + + SharedStruct entry = new SharedStruct(); + entry.key = logid; + entry.value = val.ToString(); + log[logid] = entry; + + return val; + } + + public SharedStruct getStruct(int key) + { + Console.WriteLine("getStruct({0})", key); + return log[key]; + } + + public void zip() + { + Console.WriteLine("zip()"); + } + } + + public class CSharpServer + { + public static void Main() + { + try + { + CalculatorHandler handler = new CalculatorHandler(); + Calculator.Processor processor = new Calculator.Processor(handler); + TServerTransport serverTransport = new TServerSocket(9090); + TServer server = new TSimpleServer(processor, serverTransport); + + // Use this for a multithreaded server + // server = new TThreadPoolServer(processor, serverTransport); + + Console.WriteLine("Starting the server..."); + server.Serve(); + } + catch (Exception x) + { + Console.WriteLine(x.StackTrace); + } + Console.WriteLine("done."); + } + } +} diff --git a/test/csharp/ThriftTest/Program.cs b/test/csharp/ThriftTest/Program.cs new file mode 100644 index 00000000..4c63ca47 --- /dev/null +++ b/test/csharp/ThriftTest/Program.cs @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ + +using System; +using Thrift.Transport; +using Thrift.Protocol; +using Thrift.Test; //generated code + +namespace Test +{ + class Program + { + static void Main(string[] args) + { + if (args.Length == 0) + { + Console.WriteLine("must provide 'server' or 'client' arg"); + return; + } + + string[] subArgs = new string[args.Length - 1]; + for(int i = 1; i < args.Length; i++) + { + subArgs[i-1] = args[i]; + } + if (args[0] == "client") + { + TestClient.Execute(subArgs); + } + else if (args[0] == "server") + { + TestServer.Execute(subArgs); + } + else + { + Console.WriteLine("first argument must be 'server' or 'client'"); + } + } + } +} diff --git a/test/csharp/ThriftTest/Properties/AssemblyInfo.cs b/test/csharp/ThriftTest/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..504ca8de --- /dev/null +++ b/test/csharp/ThriftTest/Properties/AssemblyInfo.cs @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("ThriftTest")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("ThriftTest")] +[assembly: AssemblyCopyright("Copyright © 2009 The Apache Software Foundation")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("f41b193b-f1ab-48ee-8843-f88e43084e26")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/test/csharp/ThriftTest/TestClient.cs b/test/csharp/ThriftTest/TestClient.cs new file mode 100644 index 00000000..2d278b7f --- /dev/null +++ b/test/csharp/ThriftTest/TestClient.cs @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using Thrift.Collections; +using Thrift.Protocol; +using Thrift.Transport; +using Thrift.Test; + +namespace Test +{ + public class TestClient + { + private static int numIterations = 1; + + public static void Execute(string[] args) + { + try + { + string host = "localhost"; + int port = 9090; + string url = null; + int numThreads = 1; + bool buffered = false; + + try + { + for (int i = 0; i < args.Length; i++) + { + if (args[i] == "-h") + { + string[] hostport = args[++i].Split(':'); + host = hostport[0]; + if (hostport.Length > 1) + { + port = Convert.ToInt32(hostport[1]); + } + } + else if (args[i] == "-u") + { + url = args[++i]; + } + else if (args[i] == "-n") + { + numIterations = Convert.ToInt32(args[++i]); + } + else if (args[i] == "-b" || args[i] == "-buffered") + { + buffered = true; + Console.WriteLine("Using buffered sockets"); + } + else if (args[i] == "-t") + { + numThreads = Convert.ToInt32(args[++i]); + } + } + } + catch (Exception e) + { + Console.WriteLine(e.StackTrace); + } + + + + //issue tests on separate threads simultaneously + Thread[] threads = new Thread[numThreads]; + DateTime start = DateTime.Now; + for (int test = 0; test < numThreads; test++) + { + Thread t = new Thread(new ParameterizedThreadStart(ClientThread)); + threads[test] = t; + TSocket socket = new TSocket(host, port); + if (buffered) + { + TBufferedTransport buffer = new TBufferedTransport(socket); + t.Start(buffer); + } + else + { + t.Start(socket); + } + } + + for (int test = 0; test < numThreads; test++) + { + threads[test].Join(); + } + Console.Write("Total time: " + (DateTime.Now - start)); + } + catch (Exception outerEx) + { + Console.WriteLine(outerEx.Message + " ST: " + outerEx.StackTrace); + } + + Console.WriteLine(); + Console.WriteLine(); + } + + public static void ClientThread(object obj) + { + TTransport transport = (TTransport)obj; + for (int i = 0; i < numIterations; i++) + { + ClientTest(transport); + } + transport.Close(); + } + + public static void ClientTest(TTransport transport) + { + TBinaryProtocol binaryProtocol = new TBinaryProtocol(transport); + + ThriftTest.Client client = new ThriftTest.Client(binaryProtocol); + try + { + if (!transport.IsOpen) + { + transport.Open(); + } + } + catch (TTransportException ttx) + { + Console.WriteLine("Connect failed: " + ttx.Message); + return; + } + + long start = DateTime.Now.ToFileTime(); + + Console.Write("testVoid()"); + client.testVoid(); + Console.WriteLine(" = void"); + + Console.Write("testString(\"Test\")"); + string s = client.testString("Test"); + Console.WriteLine(" = \"" + s + "\""); + + Console.Write("testByte(1)"); + byte i8 = client.testByte((byte)1); + Console.WriteLine(" = " + i8); + + Console.Write("testI32(-1)"); + int i32 = client.testI32(-1); + Console.WriteLine(" = " + i32); + + Console.Write("testI64(-34359738368)"); + long i64 = client.testI64(-34359738368); + Console.WriteLine(" = " + i64); + + Console.Write("testDouble(5.325098235)"); + double dub = client.testDouble(5.325098235); + Console.WriteLine(" = " + dub); + + Console.Write("testStruct({\"Zero\", 1, -3, -5})"); + Xtruct o = new Xtruct(); + o.String_thing = "Zero"; + o.Byte_thing = (byte)1; + o.I32_thing = -3; + o.I64_thing = -5; + Xtruct i = client.testStruct(o); + Console.WriteLine(" = {\"" + i.String_thing + "\", " + i.Byte_thing + ", " + i.I32_thing + ", " + i.I64_thing + "}"); + + Console.Write("testNest({1, {\"Zero\", 1, -3, -5}, 5})"); + Xtruct2 o2 = new Xtruct2(); + o2.Byte_thing = (byte)1; + o2.Struct_thing = o; + o2.I32_thing = 5; + Xtruct2 i2 = client.testNest(o2); + i = i2.Struct_thing; + Console.WriteLine(" = {" + i2.Byte_thing + ", {\"" + i.String_thing + "\", " + i.Byte_thing + ", " + i.I32_thing + ", " + i.I64_thing + "}, " + i2.I32_thing + "}"); + + Dictionary mapout = new Dictionary(); + for (int j = 0; j < 5; j++) + { + mapout[j] = j - 10; + } + Console.Write("testMap({"); + bool first = true; + foreach (int key in mapout.Keys) + { + if (first) + { + first = false; + } + else + { + Console.Write(", "); + } + Console.Write(key + " => " + mapout[key]); + } + Console.Write("})"); + + Dictionary mapin = client.testMap(mapout); + + Console.Write(" = {"); + first = true; + foreach (int key in mapin.Keys) + { + if (first) + { + first = false; + } + else + { + Console.Write(", "); + } + Console.Write(key + " => " + mapin[key]); + } + Console.WriteLine("}"); + + List listout = new List(); + for (int j = -2; j < 3; j++) + { + listout.Add(j); + } + Console.Write("testList({"); + first = true; + foreach (int j in listout) + { + if (first) + { + first = false; + } + else + { + Console.Write(", "); + } + Console.Write(j); + } + Console.Write("})"); + + List listin = client.testList(listout); + + Console.Write(" = {"); + first = true; + foreach (int j in listin) + { + if (first) + { + first = false; + } + else + { + Console.Write(", "); + } + Console.Write(j); + } + Console.WriteLine("}"); + + //set + THashSet setout = new THashSet(); + for (int j = -2; j < 3; j++) + { + setout.Add(j); + } + Console.Write("testSet({"); + first = true; + foreach (int j in setout) + { + if (first) + { + first = false; + } + else + { + Console.Write(", "); + } + Console.Write(j); + } + Console.Write("})"); + + THashSet setin = client.testSet(setout); + + Console.Write(" = {"); + first = true; + foreach (int j in setin) + { + if (first) + { + first = false; + } + else + { + Console.Write(", "); + } + Console.Write(j); + } + Console.WriteLine("}"); + + + Console.Write("testEnum(ONE)"); + Numberz ret = client.testEnum(Numberz.ONE); + Console.WriteLine(" = " + ret); + + Console.Write("testEnum(TWO)"); + ret = client.testEnum(Numberz.TWO); + Console.WriteLine(" = " + ret); + + Console.Write("testEnum(THREE)"); + ret = client.testEnum(Numberz.THREE); + Console.WriteLine(" = " + ret); + + Console.Write("testEnum(FIVE)"); + ret = client.testEnum(Numberz.FIVE); + Console.WriteLine(" = " + ret); + + Console.Write("testEnum(EIGHT)"); + ret = client.testEnum(Numberz.EIGHT); + Console.WriteLine(" = " + ret); + + Console.Write("testTypedef(309858235082523)"); + long uid = client.testTypedef(309858235082523L); + Console.WriteLine(" = " + uid); + + Console.Write("testMapMap(1)"); + Dictionary> mm = client.testMapMap(1); + Console.Write(" = {"); + foreach (int key in mm.Keys) + { + Console.Write(key + " => {"); + Dictionary m2 = mm[key]; + foreach (int k2 in m2.Keys) + { + Console.Write(k2 + " => " + m2[k2] + ", "); + } + Console.Write("}, "); + } + Console.WriteLine("}"); + + Insanity insane = new Insanity(); + insane.UserMap = new Dictionary(); + insane.UserMap[Numberz.FIVE] = 5000L; + Xtruct truck = new Xtruct(); + truck.String_thing = "Truck"; + truck.Byte_thing = (byte)8; + truck.I32_thing = 8; + truck.I64_thing = 8; + insane.Xtructs = new List(); + insane.Xtructs.Add(truck); + Console.Write("testInsanity()"); + Dictionary> whoa = client.testInsanity(insane); + Console.Write(" = {"); + foreach (long key in whoa.Keys) + { + Dictionary val = whoa[key]; + Console.Write(key + " => {"); + + foreach (Numberz k2 in val.Keys) + { + Insanity v2 = val[k2]; + + Console.Write(k2 + " => {"); + Dictionary userMap = v2.UserMap; + + Console.Write("{"); + if (userMap != null) + { + foreach (Numberz k3 in userMap.Keys) + { + Console.Write(k3 + " => " + userMap[k3] + ", "); + } + } + else + { + Console.Write("null"); + } + Console.Write("}, "); + + List xtructs = v2.Xtructs; + + Console.Write("{"); + if (xtructs != null) + { + foreach (Xtruct x in xtructs) + { + Console.Write("{\"" + x.String_thing + "\", " + x.Byte_thing + ", " + x.I32_thing + ", " + x.I32_thing + "}, "); + } + } + else + { + Console.Write("null"); + } + Console.Write("}"); + + Console.Write("}, "); + } + Console.Write("}, "); + } + Console.WriteLine("}"); + + + byte arg0 = 1; + int arg1 = 2; + long arg2 = long.MaxValue; + Dictionary multiDict = new Dictionary(); + multiDict[1] = "one"; + Numberz arg4 = Numberz.FIVE; + long arg5 = 5000000; + Console.Write("Test Multi(" + arg0 + "," + arg1 + "," + arg2 + "," + multiDict + "," + arg4 + "," + arg5 + ")"); + Xtruct multiResponse = client.testMulti(arg0, arg1, arg2, multiDict, arg4, arg5); + Console.Write(" = Xtruct(byte_thing:" + multiResponse.Byte_thing + ",String_thing:" + multiResponse.String_thing + + ",i32_thing:" + multiResponse.I32_thing + ",i64_thing:" + multiResponse.I64_thing + ")\n"); + + Console.WriteLine("Test Oneway(1)"); + client.testOneway(1); + } + } +} diff --git a/test/csharp/ThriftTest/TestServer.cs b/test/csharp/ThriftTest/TestServer.cs new file mode 100644 index 00000000..e3706404 --- /dev/null +++ b/test/csharp/ThriftTest/TestServer.cs @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Distributed under the Thrift Software License +// +// See accompanying file LICENSE or visit the Thrift site at: +// http://developers.facebook.com/thrift/ +using System; +using System.Collections.Generic; +using Thrift.Collections; +using Thrift.Test; //generated code +using Thrift.Transport; +using Thrift.Protocol; +using Thrift.Server; + +namespace Test +{ + public class TestServer + { + public class TestHandler : ThriftTest.Iface + { + public TServer server; + + public TestHandler() { } + + public void testVoid() + { + Console.WriteLine("testVoid()"); + } + + public string testString(string thing) + { + Console.WriteLine("teststring(\"" + thing + "\")"); + return thing; + } + + public byte testByte(byte thing) + { + Console.WriteLine("testByte(" + thing + ")"); + return thing; + } + + public int testI32(int thing) + { + Console.WriteLine("testI32(" + thing + ")"); + return thing; + } + + public long testI64(long thing) + { + Console.WriteLine("testI64(" + thing + ")"); + return thing; + } + + public double testDouble(double thing) + { + Console.WriteLine("testDouble(" + thing + ")"); + return thing; + } + + public Xtruct testStruct(Xtruct thing) + { + Console.WriteLine("testStruct({" + + "\"" + thing.String_thing + "\", " + + thing.Byte_thing + ", " + + thing.I32_thing + ", " + + thing.I64_thing + "})"); + return thing; + } + + public Xtruct2 testNest(Xtruct2 nest) + { + Xtruct thing = nest.Struct_thing; + Console.WriteLine("testNest({" + + nest.Byte_thing + ", {" + + "\"" + thing.String_thing + "\", " + + thing.Byte_thing + ", " + + thing.I32_thing + ", " + + thing.I64_thing + "}, " + + nest.I32_thing + "})"); + return nest; + } + + public Dictionary testMap(Dictionary thing) + { + Console.WriteLine("testMap({"); + bool first = true; + foreach (int key in thing.Keys) + { + if (first) + { + first = false; + } + else + { + Console.WriteLine(", "); + } + Console.WriteLine(key + " => " + thing[key]); + } + Console.WriteLine("})"); + return thing; + } + + public THashSet testSet(THashSet thing) + { + Console.WriteLine("testSet({"); + bool first = true; + foreach (int elem in thing) + { + if (first) + { + first = false; + } + else + { + Console.WriteLine(", "); + } + Console.WriteLine(elem); + } + Console.WriteLine("})"); + return thing; + } + + public List testList(List thing) + { + Console.WriteLine("testList({"); + bool first = true; + foreach (int elem in thing) + { + if (first) + { + first = false; + } + else + { + Console.WriteLine(", "); + } + Console.WriteLine(elem); + } + Console.WriteLine("})"); + return thing; + } + + public Numberz testEnum(Numberz thing) + { + Console.WriteLine("testEnum(" + thing + ")"); + return thing; + } + + public long testTypedef(long thing) + { + Console.WriteLine("testTypedef(" + thing + ")"); + return thing; + } + + public Dictionary> testMapMap(int hello) + { + Console.WriteLine("testMapMap(" + hello + ")"); + Dictionary> mapmap = + new Dictionary>(); + + Dictionary pos = new Dictionary(); + Dictionary neg = new Dictionary(); + for (int i = 1; i < 5; i++) + { + pos[i] = i; + neg[-i] = -i; + } + + mapmap[4] = pos; + mapmap[-4] = neg; + + return mapmap; + } + + public Dictionary> testInsanity(Insanity argument) + { + Console.WriteLine("testInsanity()"); + + Xtruct hello = new Xtruct(); + hello.String_thing = "Hello2"; + hello.Byte_thing = 2; + hello.I32_thing = 2; + hello.I64_thing = 2; + + Xtruct goodbye = new Xtruct(); + goodbye.String_thing = "Goodbye4"; + goodbye.Byte_thing = (byte)4; + goodbye.I32_thing = 4; + goodbye.I64_thing = (long)4; + + Insanity crazy = new Insanity(); + crazy.UserMap = new Dictionary(); + crazy.UserMap[Numberz.EIGHT] = (long)8; + crazy.Xtructs = new List(); + crazy.Xtructs.Add(goodbye); + + Insanity looney = new Insanity(); + crazy.UserMap[Numberz.FIVE] = (long)5; + crazy.Xtructs.Add(hello); + + Dictionary first_map = new Dictionary(); + Dictionary second_map = new Dictionary(); ; + + first_map[Numberz.TWO] = crazy; + first_map[Numberz.THREE] = crazy; + + second_map[Numberz.SIX] = looney; + + Dictionary> insane = + new Dictionary>(); + insane[(long)1] = first_map; + insane[(long)2] = second_map; + + return insane; + } + + public Xtruct testMulti(byte arg0, int arg1, long arg2, Dictionary arg3, Numberz arg4, long arg5) + { + Console.WriteLine("testMulti()"); + + Xtruct hello = new Xtruct(); ; + hello.String_thing = "Hello2"; + hello.Byte_thing = arg0; + hello.I32_thing = arg1; + hello.I64_thing = arg2; + return hello; + } + + public void testException(string arg) + { + Console.WriteLine("testException(" + arg + ")"); + if (arg == "Xception") + { + Xception x = new Xception(); + x.ErrorCode = 1001; + x.Message = "This is an Xception"; + throw x; + } + return; + } + + public Xtruct testMultiException(string arg0, string arg1) + { + Console.WriteLine("testMultiException(" + arg0 + ", " + arg1 + ")"); + if (arg0 == "Xception") + { + Xception x = new Xception(); + x.ErrorCode = 1001; + x.Message = "This is an Xception"; + throw x; + } + else if (arg0 == "Xception2") + { + Xception2 x = new Xception2(); + x.ErrorCode = 2002; + x.Struct_thing = new Xtruct(); + x.Struct_thing.String_thing = "This is an Xception2"; + throw x; + } + + Xtruct result = new Xtruct(); + result.String_thing = arg1; + return result; + } + + public void testStop() + { + if (server != null) + { + server.Stop(); + } + } + + public void testOneway(int arg) + { + Console.WriteLine("testOneway(" + arg + "), sleeping..."); + System.Threading.Thread.Sleep(arg * 1000); + Console.WriteLine("testOneway finished"); + } + + } // class TestHandler + + public static void Execute(string[] args) + { + try + { + bool useBufferedSockets = false; + int port = 9090; + if (args.Length > 0) + { + port = int.Parse(args[0]); + + if (args.Length > 1) + { + bool.TryParse(args[1], out useBufferedSockets); + } + } + + // Processor + TestHandler testHandler = new TestHandler(); + ThriftTest.Processor testProcessor = new ThriftTest.Processor(testHandler); + + // Transport + TServerSocket tServerSocket = new TServerSocket(port, 0, useBufferedSockets); + + TServer serverEngine; + + // Simple Server + serverEngine = new TSimpleServer(testProcessor, tServerSocket); + + // ThreadPool Server + // serverEngine = new TThreadPoolServer(testProcessor, tServerSocket); + + // Threaded Server + // serverEngine = new TThreadedServer(testProcessor, tServerSocket); + + testHandler.server = serverEngine; + + // Run it + Console.WriteLine("Starting the server on port " + port + (useBufferedSockets ? " with buffered socket" : "") + "..."); + serverEngine.Serve(); + + } + catch (Exception x) + { + Console.Error.Write(x); + } + Console.WriteLine("done."); + } + } +} diff --git a/test/csharp/ThriftTest/ThriftTest.csproj b/test/csharp/ThriftTest/ThriftTest.csproj new file mode 100644 index 00000000..3f427fd7 --- /dev/null +++ b/test/csharp/ThriftTest/ThriftTest.csproj @@ -0,0 +1,111 @@ + + + + Debug + AnyCPU + 9.0.21022 + 2.0 + {48DD757F-CA95-4DD7-BDA4-58DB6F108C2C} + Exe + Properties + ThriftTest + ThriftTest + v3.5 + 512 + publish\ + true + Disk + false + Foreground + 7 + Days + false + false + true + 0 + 1.0.0.%2a + false + false + true + SAK + SAK + SAK + SAK + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + False + .\ThriftImpl.dll + + + + + + + + + + + False + .NET Framework 2.0 %28x86%29 + false + + + False + .NET Framework 3.0 %28x86%29 + false + + + False + .NET Framework 3.5 + true + + + False + Windows Installer 3.1 + true + + + + + {499EB63C-D74C-47E8-AE48-A2FC94538E9D} + Thrift + + + + + + rmdir /s /q $(ProjectDir)gen-csharp +del /f /q $(ProjectDir)ThriftImpl.dll + +$(ProjectDir)\..\..\..\compiler\cpp\thrift.exe -csharp -o $(ProjectDir) $(ProjectDir)\..\..\ThriftTest.thrift + +cd $(ProjectDir) + +$(MSBuildToolsPath)\Csc.exe /t:library /out:.\ThriftImpl.dll /recurse:.\gen-csharp\* /reference:$(ProjectDir)..\..\..\lib\csharp\src\bin\Debug\Thrift.dll + + diff --git a/test/csharp/ThriftTest/maketest.sh b/test/csharp/ThriftTest/maketest.sh new file mode 100755 index 00000000..5580de80 --- /dev/null +++ b/test/csharp/ThriftTest/maketest.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +../../../compiler/cpp/thrift --gen csharp -o . ../../ThriftTest.thrift +gmcs /t:library /out:./ThriftImpl.dll /recurse:./gen-csharp/* /reference:../../../lib/csharp/Thrift.dll diff --git a/test/erl/Makefile b/test/erl/Makefile new file mode 100644 index 00000000..17e30da6 --- /dev/null +++ b/test/erl/Makefile @@ -0,0 +1,66 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +GENDIR=gen +GEN_INCLUDEDIR=$(GENDIR)/include +GEN_SRCDIR=$(GENDIR)/src +GEN_TARGETDIR=$(GENDIR)/ebin + +INCLUDEDIR=include +TARGETDIR=ebin +SRCDIR=src + +ALL_INCLUDEDIR=$(GEN_INCLUDEDIR) $(INCLUDEDIR) ../../lib/erl/include +INCLUDEFLAGS=$(patsubst %,-I%, ${ALL_INCLUDEDIR}) + +MODULES = stress_server test_server test_disklog test_membuffer + +INCLUDES = +TARGETS = $(patsubst %,${TARGETDIR}/%.beam,${MODULES}) +HEADERS = $(patsubst %,${INCLUDEDIR}/%.hrl,${INCLUDES}) + +all: ${GEN_TARGETDIR}/ ${TARGETS} + +TEST_RPCFILE = ../ThriftTest.thrift +STRESS_RPCFILE = ../StressTest.thrift +THRIFT = ../../compiler/cpp/thrift + +${GENDIR}/: ${RPCFILE} + rm -rf ${GENDIR} + ${THRIFT} --gen erl ${TEST_RPCFILE} + ${THRIFT} --gen erl ${STRESS_RPCFILE} + mkdir -p ${GEN_INCLUDEDIR} + mkdir -p ${GEN_SRCDIR} + mkdir -p ${GEN_TARGETDIR} + mv -t ${GEN_INCLUDEDIR} gen-erl/*.hrl + mv -t ${GEN_SRCDIR} gen-erl/*.erl + rm -rf gen-erl + +${GEN_TARGETDIR}/: ${GENDIR}/ + rm -rf ${GEN_TARGETDIR} + mkdir -p ${GEN_TARGETDIR} + erlc ${INCLUDEFLAGS} -o ${GEN_TARGETDIR} ${GEN_SRCDIR}/*.erl + +$(TARGETS): ${TARGETDIR}/%.beam: ${SRCDIR}/%.erl ${GEN_INCLUDEDIR}/ ${HEADERS} + mkdir -p ${TARGETDIR} + erlc ${INCLUDEFLAGS} -o ${TARGETDIR} $< + +clean: + rm -f ${TARGETDIR}/*.beam + rm -rf ${GENDIR} diff --git a/test/erl/src/stress_server.erl b/test/erl/src/stress_server.erl new file mode 100644 index 00000000..35fff069 --- /dev/null +++ b/test/erl/src/stress_server.erl @@ -0,0 +1,64 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(stress_server). + + +-export([start_link/1, + + handle_function/2, + + echoVoid/0, + echoByte/1, + echoI32/1, + echoI64/1, + echoString/1, + echoList/1, + echoSet/1, + echoMap/1 + ]). + +start_link(Port) -> + thrift_server:start_link(Port, service_thrift, ?MODULE). + + +handle_function(Function, Args) -> + case apply(?MODULE, Function, tuple_to_list(Args)) of + ok -> + ok; + Else -> {reply, Else} + end. + + +echoVoid() -> + ok. +echoByte(X) -> + X. +echoI32(X) -> + X. +echoI64(X) -> + X. +echoString(X) -> + X. +echoList(X) -> + X. +echoSet(X) -> + X. +echoMap(X) -> + X. diff --git a/test/erl/src/test_disklog.erl b/test/erl/src/test_disklog.erl new file mode 100644 index 00000000..7b0be72d --- /dev/null +++ b/test/erl/src/test_disklog.erl @@ -0,0 +1,81 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(test_disklog). + +-compile(export_all). + +t() -> + {ok, TransportFactory} = + thrift_disk_log_transport:new_transport_factory( + test_disklog, + [{file, "/tmp/test_log"}, + {size, {1024*1024, 10}}]), + {ok, ProtocolFactory} = thrift_binary_protocol:new_protocol_factory( + TransportFactory, []), + {ok, Client} = thrift_client:start_link(ProtocolFactory, thriftTest_thrift), + + io:format("Client started~n"), + + % We have to make oneway calls into this client only since otherwise it will try + % to read from the disklog and go boom. + {ok, ok} = thrift_client:call(Client, testOneway, [16#deadbeef]), + io:format("Call written~n"), + + % Use the send_call method to write a non-oneway call into the log + ok = thrift_client:send_call(Client, testString, [<<"hello world">>]), + io:format("Non-oneway call sent~n"), + + ok = thrift_client:close(Client), + io:format("Client closed~n"), + + ok. + + + +t_base64() -> + {ok, TransportFactory} = + thrift_disk_log_transport:new_transport_factory( + test_disklog, + [{file, "/tmp/test_b64_log"}, + {size, {1024*1024, 10}}]), + {ok, B64Factory} = + thrift_base64_transport:new_transport_factory(TransportFactory), + {ok, BufFactory} = + thrift_buffered_transport:new_transport_factory(B64Factory), + {ok, ProtocolFactory} = thrift_binary_protocol:new_protocol_factory( + BufFactory, []), + {ok, Client} = thrift_client:start_link(ProtocolFactory, thriftTest_thrift), + + io:format("Client started~n"), + + % We have to make oneway calls into this client only since otherwise it will try + % to read from the disklog and go boom. + {ok, ok} = thrift_client:call(Client, testOneway, [16#deadbeef]), + io:format("Call written~n"), + + % Use the send_call method to write a non-oneway call into the log + ok = thrift_client:send_call(Client, testString, [<<"hello world">>]), + io:format("Non-oneway call sent~n"), + + ok = thrift_client:close(Client), + io:format("Client closed~n"), + + ok. + diff --git a/test/erl/src/test_membuffer.erl b/test/erl/src/test_membuffer.erl new file mode 100644 index 00000000..7bd23a0f --- /dev/null +++ b/test/erl/src/test_membuffer.erl @@ -0,0 +1,81 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(test_membuffer). +-export([t/0]). + +-include("thriftTest_types.hrl"). + +test_data() -> + #xtruct{string_thing = <<"foobar">>, + byte_thing = 123, + i32_thing = 1234567, + i64_thing = 12345678900}. + +t1() -> + {ok, Transport} = thrift_memory_buffer:new(), + {ok, Protocol} = thrift_binary_protocol:new(Transport), + TestData = test_data(), + ok = thrift_protocol:write(Protocol, + {{struct, element(2, thriftTest_types:struct_info('xtruct'))}, + TestData}), + {ok, Result} = thrift_protocol:read(Protocol, + {struct, element(2, thriftTest_types:struct_info('xtruct'))}, + 'xtruct'), + + Result = TestData. + + +t2() -> + {ok, Transport} = thrift_memory_buffer:new(), + {ok, Protocol} = thrift_binary_protocol:new(Transport), + TestData = test_data(), + ok = thrift_protocol:write(Protocol, + {{struct, element(2, thriftTest_types:struct_info('xtruct'))}, + TestData}), + {ok, Result} = thrift_protocol:read(Protocol, + {struct, element(2, thriftTest_types:struct_info('xtruct3'))}, + 'xtruct3'), + + Result = #xtruct3{string_thing = TestData#xtruct.string_thing, + changed = undefined, + i32_thing = TestData#xtruct.i32_thing, + i64_thing = TestData#xtruct.i64_thing}. + + +t3() -> + {ok, Transport} = thrift_memory_buffer:new(), + {ok, Protocol} = thrift_binary_protocol:new(Transport), + TestData = #bools{im_true = true, im_false = false}, + ok = thrift_protocol:write(Protocol, + {{struct, element(2, thriftTest_types:struct_info('bools'))}, + TestData}), + {ok, Result} = thrift_protocol:read(Protocol, + {struct, element(2, thriftTest_types:struct_info('bools'))}, + 'bools'), + + true = TestData#bools.im_true =:= Result#bools.im_true, + true = TestData#bools.im_false =:= Result#bools.im_false. + + +t() -> + t1(), + t2(), + t3(). + diff --git a/test/erl/src/test_server.erl b/test/erl/src/test_server.erl new file mode 100644 index 00000000..cd439ccd --- /dev/null +++ b/test/erl/src/test_server.erl @@ -0,0 +1,174 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(test_server). + +-export([start_link/1, handle_function/2]). + +-include("thriftTest_types.hrl"). + +start_link(Port) -> + thrift_server:start_link(Port, thriftTest_thrift, ?MODULE). + + +handle_function(testVoid, {}) -> + io:format("testVoid~n"), + ok; + +handle_function(testString, {S}) when is_binary(S) -> + io:format("testString: ~p~n", [S]), + {reply, S}; + +handle_function(testByte, {I8}) when is_integer(I8) -> + io:format("testByte: ~p~n", [I8]), + {reply, I8}; + +handle_function(testI32, {I32}) when is_integer(I32) -> + io:format("testI32: ~p~n", [I32]), + {reply, I32}; + +handle_function(testI64, {I64}) when is_integer(I64) -> + io:format("testI64: ~p~n", [I64]), + {reply, I64}; + +handle_function(testDouble, {Double}) when is_float(Double) -> + io:format("testDouble: ~p~n", [Double]), + {reply, Double}; + +handle_function(testStruct, + {Struct = #xtruct{string_thing = String, + byte_thing = Byte, + i32_thing = I32, + i64_thing = I64}}) +when is_binary(String), + is_integer(Byte), + is_integer(I32), + is_integer(I64) -> + io:format("testStruct: ~p~n", [Struct]), + {reply, Struct}; + +handle_function(testNest, + {Nest}) when is_record(Nest, xtruct2), + is_record(Nest#xtruct2.struct_thing, xtruct) -> + io:format("testNest: ~p~n", [Nest]), + {reply, Nest}; + +handle_function(testMap, {Map}) -> + io:format("testMap: ~p~n", [dict:to_list(Map)]), + {reply, Map}; + +handle_function(testSet, {Set}) -> + true = sets:is_set(Set), + io:format("testSet: ~p~n", [sets:to_list(Set)]), + {reply, Set}; + +handle_function(testList, {List}) when is_list(List) -> + io:format("testList: ~p~n", [List]), + {reply, List}; + +handle_function(testEnum, {Enum}) when is_integer(Enum) -> + io:format("testEnum: ~p~n", [Enum]), + {reply, Enum}; + +handle_function(testTypedef, {UserID}) when is_integer(UserID) -> + io:format("testTypedef: ~p~n", [UserID]), + {reply, UserID}; + +handle_function(testMapMap, {Hello}) -> + io:format("testMapMap: ~p~n", [Hello]), + + PosList = [{I, I} || I <- lists:seq(1, 5)], + NegList = [{-I, -I} || I <- lists:seq(1, 5)], + + MapMap = dict:from_list([{4, dict:from_list(PosList)}, + {-4, dict:from_list(NegList)}]), + {reply, MapMap}; + +handle_function(testInsanity, {Insanity}) when is_record(Insanity, insanity) -> + Hello = #xtruct{string_thing = <<"Hello2">>, + byte_thing = 2, + i32_thing = 2, + i64_thing = 2}, + + Goodbye = #xtruct{string_thing = <<"Goodbye4">>, + byte_thing = 4, + i32_thing = 4, + i64_thing = 4}, + Crazy = #insanity{ + userMap = dict:from_list([{?thriftTest_EIGHT, 8}]), + xtructs = [Goodbye] + }, + + Looney = #insanity{ + userMap = dict:from_list([{?thriftTest_FIVE, 5}]), + xtructs = [Hello] + }, + + FirstMap = dict:from_list([{?thriftTest_TWO, Crazy}, + {?thriftTest_THREE, Crazy}]), + + SecondMap = dict:from_list([{?thriftTest_SIX, Looney}]), + + Insane = dict:from_list([{1, FirstMap}, + {2, SecondMap}]), + + io:format("Return = ~p~n", [Insane]), + + {reply, Insane}; + +handle_function(testMulti, Args = {Arg0, Arg1, Arg2, _Arg3, Arg4, Arg5}) + when is_integer(Arg0), + is_integer(Arg1), + is_integer(Arg2), + is_integer(Arg4), + is_integer(Arg5) -> + + io:format("testMulti(~p)~n", [Args]), + {reply, #xtruct{string_thing = <<"Hello2">>, + byte_thing = Arg0, + i32_thing = Arg1, + i64_thing = Arg2}}; + +handle_function(testException, {String}) when is_binary(String) -> + io:format("testException(~p)~n", [String]), + case String of + <<"Xception">> -> + throw(#xception{errorCode = 1001, + message = <<"This is an Xception">>}); + _ -> + ok + end; + +handle_function(testMultiException, {Arg0, Arg1}) -> + io:format("testMultiException(~p, ~p)~n", [Arg0, Arg1]), + case Arg0 of + <<"Xception">> -> + throw(#xception{errorCode = 1001, + message = <<"This is an Xception">>}); + <<"Xception2">> -> + throw(#xception2{errorCode = 2002, + struct_thing = + #xtruct{string_thing = <<"This is an Xception2">>}}); + _ -> + {reply, #xtruct{string_thing = Arg1}} + end; + +handle_function(testOneway, {Seconds}) -> + timer:sleep(1000 * Seconds), + ok. diff --git a/test/hs/Client.hs b/test/hs/Client.hs new file mode 100644 index 00000000..c5e4d907 --- /dev/null +++ b/test/hs/Client.hs @@ -0,0 +1,58 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +module Client where + +import ThriftTest_Client +import ThriftTest_Types +import qualified Data.Map as Map +import qualified Data.Set as Set +import Control.Monad +import Control.Exception as CE + +import Network + +import Thrift +import Thrift.Transport.Handle +import Thrift.Protocol.Binary + + +serverAddress = ("127.0.0.1", PortNumber 9090) + +main = do to <- hOpen serverAddress + let p = BinaryProtocol to + let ps = (p,p) + print =<< testString ps "bya" + print =<< testByte ps 8 + print =<< testByte ps (-8) + print =<< testI32 ps 32 + print =<< testI32 ps (-32) + print =<< testI64 ps 64 + print =<< testI64 ps (-64) + print =<< testDouble ps 3.14 + print =<< testDouble ps (-3.14) + print =<< testMap ps (Map.fromList [(1,1),(2,2),(3,3)]) + print =<< testList ps [1,2,3,4,5] + print =<< testSet ps (Set.fromList [1,2,3,4,5]) + print =<< testStruct ps (Xtruct (Just "hi") (Just 4) (Just 5) Nothing) + CE.catch (testException ps "e" >> print "bad") (\e -> print (e :: Xception)) + CE.catch (testMultiException ps "e" "e2" >> print "ok") (\e -> print (e :: Xception)) + CE.catch (CE.catch (testMultiException ps "e" "e2">> print "bad") (\e -> print (e :: Xception2))) (\(e :: SomeException) -> print "ok") + tClose to + diff --git a/test/hs/Server.hs b/test/hs/Server.hs new file mode 100644 index 00000000..0ca9d9fe --- /dev/null +++ b/test/hs/Server.hs @@ -0,0 +1,57 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +module Server where + +import ThriftTest +import ThriftTest_Iface +import Data.Map as Map +import Control.Exception +import ThriftTest_Types + +import Thrift +import Thrift.Server + + +data TestHandler = TestHandler +instance ThriftTest_Iface TestHandler where + testVoid a = return () + testString a (Just s) = do print s; return s + testByte a (Just x) = do print x; return x + testI32 a (Just x) = do print x; return x + testI64 a (Just x) = do print x; return x + testDouble a (Just x) = do print x; return x + testStruct a (Just x) = do print x; return x + testNest a (Just x) = do print x; return x + testMap a (Just x) = do print x; return x + testSet a (Just x) = do print x; return x + testList a (Just x) = do print x; return x + testEnum a (Just x) = do print x; return x + testTypedef a (Just x) = do print x; return x + testMapMap a (Just x) = return (Map.fromList [(1,Map.fromList [(2,2)])]) + testInsanity a (Just x) = return (Map.fromList [(1,Map.fromList [(ONE,x)])]) + testMulti a a1 a2 a3 a4 a5 a6 = return (Xtruct Nothing Nothing Nothing Nothing) + testException a c = throw (Xception (Just 1) (Just "bya")) + testMultiException a c1 c2 = throw (Xception (Just 1) (Just "xyz")) + testOneway a (Just i) = do print i + + +main = do (runBasicServer TestHandler process 9090) + `Control.Exception.catch` + (\(TransportExn s t) -> print s) diff --git a/test/hs/runclient.sh b/test/hs/runclient.sh new file mode 100644 index 00000000..b93bbb14 --- /dev/null +++ b/test/hs/runclient.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +if [ -z $BASE ]; then + BASE=../.. +fi + +ghci -fglasgow-exts -i$BASE/lib/hs/src -i$BASE/test/hs/gen-hs Client.hs diff --git a/test/hs/runserver.sh b/test/hs/runserver.sh new file mode 100644 index 00000000..b23301b4 --- /dev/null +++ b/test/hs/runserver.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +if [ -z $BASE ]; then + BASE=../.. +fi + +printf "Starting server... " +ghc -fglasgow-exts -i$BASE/lib/hs/src -i$BASE/test/hs/gen-hs Server.hs -e "putStrLn \"ready.\" >> Server.main" diff --git a/test/ocaml/Makefile b/test/ocaml/Makefile new file mode 100644 index 00000000..a543ce58 --- /dev/null +++ b/test/ocaml/Makefile @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +all: + cd client; make; cd ..; cd server; make +clean: + cd client; make clean; cd ..; cd server; make clean + diff --git a/test/ocaml/client/Makefile b/test/ocaml/client/Makefile new file mode 100644 index 00000000..806ed20a --- /dev/null +++ b/test/ocaml/client/Makefile @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +SOURCES = ../gen-ocaml/ThriftTest_types.ml ../gen-ocaml/ThriftTest_consts.ml ../gen-ocaml/SecondService.ml ../gen-ocaml/ThriftTest.ml TestClient.ml +RESULT = tc +INCDIRS = "../../../lib/ocaml/src/" "../gen-ocaml/" +LIBS = unix thrift +all: nc +OCAMLMAKEFILE = ../../../lib/ocaml/OCamlMakefile +include $(OCAMLMAKEFILE) diff --git a/test/ocaml/client/TestClient.ml b/test/ocaml/client/TestClient.ml new file mode 100644 index 00000000..91783ae4 --- /dev/null +++ b/test/ocaml/client/TestClient.ml @@ -0,0 +1,82 @@ +(* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*) + +open Thrift;; +open ThriftTest_types;; + +let s = new TSocket.t "127.0.0.1" 9090;; +let p = new TBinaryProtocol.t s;; +let c = new ThriftTest.client p p;; +let sod = function + Some v -> v + | None -> raise Thrift_error;; + +s#opn; +print_string (c#testString "bya"); +print_char '\n'; +print_int (c#testByte 8); +print_char '\n'; +print_int (c#testByte (-8)); +print_char '\n'; +print_int (c#testI32 32); +print_char '\n'; +print_string (Int64.to_string (c#testI64 64L)); +print_char '\n'; +print_float (c#testDouble 3.14); +print_char '\n'; + +let l = [1;2;3;4] in + if l = (c#testList l) then print_string "list ok\n" else print_string "list fail\n";; +let h = Hashtbl.create 5 in +let a = Hashtbl.add h in + for i=1 to 10 do + a i (10*i) + done; + let r = c#testMap h in + for i=1 to 10 do + try + let g = Hashtbl.find r i in + print_int i; + print_char ' '; + print_int g; + print_char '\n' + with Not_found -> print_string ("Can't find "^(string_of_int i)^"\n") + done;; + +let s = Hashtbl.create 5 in +let a = Hashtbl.add s in + for i = 1 to 10 do + a i true + done; + let r = c#testSet s in + for i = 1 to 10 do + try + let g = Hashtbl.find r i in + print_int i; + print_char '\n' + with Not_found -> print_string ("Can't find "^(string_of_int i)^"\n") + done;; +try + c#testException "Xception" +with Xception _ -> print_string "testException ok\n";; +try + ignore(c#testMultiException "Xception" "bya") +with Xception e -> Printf.printf "%d %s\n" (sod e#get_errorCode) (sod e#get_message);; + + diff --git a/test/ocaml/server/Makefile b/test/ocaml/server/Makefile new file mode 100644 index 00000000..44dcac76 --- /dev/null +++ b/test/ocaml/server/Makefile @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +SOURCES = ../gen-ocaml/ThriftTest_types.ml ../gen-ocaml/ThriftTest_consts.ml ../gen-ocaml/SecondService.ml ../gen-ocaml/ThriftTest.ml TestServer.ml +RESULT = ts +INCDIRS = "../../../lib/ocaml/src/" "../gen-ocaml/" +LIBS = thrift +THREADS = yes +all: nc +OCAMLMAKEFILE = ../../../lib/ocaml/OCamlMakefile +include $(OCAMLMAKEFILE) diff --git a/test/ocaml/server/TestServer.ml b/test/ocaml/server/TestServer.ml new file mode 100644 index 00000000..3f5c9ee1 --- /dev/null +++ b/test/ocaml/server/TestServer.ml @@ -0,0 +1,136 @@ +(* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*) + +open Thrift +open ThriftTest_types + +let p = Printf.printf;; +exception Die;; +let sod = function + Some v -> v + | None -> raise Die;; + + +class test_handler = +object (self) + inherit ThriftTest.iface + method testVoid = p "testVoid()\n" + method testString x = p "testString(%s)\n" (sod x); (sod x) + method testByte x = p "testByte(%d)\n" (sod x); (sod x) + method testI32 x = p "testI32(%d)\n" (sod x); (sod x) + method testI64 x = p "testI64(%s)\n" (Int64.to_string (sod x)); (sod x) + method testDouble x = p "testDouble(%f)\n" (sod x); (sod x) + method testStruct x = p "testStruct(---)\n"; (sod x) + method testNest x = p "testNest(---)\n"; (sod x) + method testMap x = p "testMap(---)\n"; (sod x) + method testSet x = p "testSet(---)\n"; (sod x) + method testList x = p "testList(---)\n"; (sod x) + method testEnum x = p "testEnum(---)\n"; (sod x) + method testTypedef x = p "testTypedef(---)\n"; (sod x) + method testMapMap x = p "testMapMap(%d)\n" (sod x); + let mm = Hashtbl.create 3 in + let pos = Hashtbl.create 7 in + let neg = Hashtbl.create 7 in + for i=1 to 4 do + Hashtbl.add pos i i; + Hashtbl.add neg (-i) (-i); + done; + Hashtbl.add mm 4 pos; + Hashtbl.add mm (-4) neg; + mm + method testInsanity x = p "testInsanity()\n"; + p "testinsanity()\n"; + let hello = new xtruct in + let goodbye = new xtruct in + let crazy = new insanity in + let looney = new insanity in + let cumap = Hashtbl.create 7 in + let insane = Hashtbl.create 7 in + let firstmap = Hashtbl.create 7 in + let secondmap = Hashtbl.create 7 in + hello#set_string_thing "Hello2"; + hello#set_byte_thing 2; + hello#set_i32_thing 2; + hello#set_i64_thing 2L; + goodbye#set_string_thing "Goodbye4"; + goodbye#set_byte_thing 4; + goodbye#set_i32_thing 4; + goodbye#set_i64_thing 4L; + Hashtbl.add cumap Numberz.EIGHT 8L; + Hashtbl.add cumap Numberz.FIVE 5L; + crazy#set_userMap cumap; + crazy#set_xtructs [goodbye; hello]; + Hashtbl.add firstmap Numberz.TWO crazy; + Hashtbl.add firstmap Numberz.THREE crazy; + Hashtbl.add secondmap Numberz.SIX looney; + Hashtbl.add insane 1L firstmap; + Hashtbl.add insane 2L secondmap; + insane + method testMulti a0 a1 a2 a3 a4 a5 = + p "testMulti()\n"; + let hello = new xtruct in + hello#set_string_thing "Hello2"; + hello#set_byte_thing (sod a0); + hello#set_i32_thing (sod a1); + hello#set_i64_thing (sod a2); + hello + method testException s = + p "testException(%S)\n" (sod s); + if (sod s) = "Xception" then + let x = new xception in + x#set_errorCode 1001; + x#set_message "This is an Xception"; + raise (Xception x) + else () + method testMultiException a0 a1 = + p "testMultiException(%S, %S)\n" (sod a0) (sod a1); + if (sod a0) = "Xception" then + let x = new xception in + x#set_errorCode 1001; + x#set_message "This is an Xception"; + raise (Xception x) + else (if (sod a0) = "Xception2" then + let x = new xception2 in + let s = new xtruct in + x#set_errorCode 2002; + s#set_string_thing "This as an Xception2"; + x#set_struct_thing s; + raise (Xception2 x) + else ()); + let res = new xtruct in + res#set_string_thing (sod a1); + res + method testOneway i = + Unix.sleep (sod i) +end;; + +let h = new test_handler in +let proc = new ThriftTest.processor h in +let port = 9090 in +let pf = new TBinaryProtocol.factory in +let server = new TThreadedServer.t + proc + (new TServerSocket.t port) + (new Transport.factory) + pf + pf +in + server#serve + + diff --git a/test/perl/Makefile b/test/perl/Makefile new file mode 100644 index 00000000..e2d81d45 --- /dev/null +++ b/test/perl/Makefile @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Default target is everything +target: all + +# Tools +THRIFT = ../../compiler/cpp/thrift + +all: ../ThriftTest.thrift + $(THRIFT) --gen perl ../ThriftTest.thrift + +clean: + $(RM) -r gen-perl diff --git a/test/perl/TestClient.pl b/test/perl/TestClient.pl new file mode 100644 index 00000000..af80d469 --- /dev/null +++ b/test/perl/TestClient.pl @@ -0,0 +1,338 @@ +#!/usr/bin/env perl + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 5.6.0; +use strict; +use warnings; +use Data::Dumper; +use Time::HiRes qw(gettimeofday); + +use lib '../../lib/perl/lib'; +use lib 'gen-perl'; + +use Thrift; +use Thrift::BinaryProtocol; +use Thrift::Socket; +use Thrift::BufferedTransport; + +use ThriftTest::ThriftTest; +use ThriftTest::Types; + +$|++; + +my $host = 'localhost'; +my $port = 9090; + + +my $socket = new Thrift::Socket($host, $port); + +my $bufferedSocket = new Thrift::BufferedTransport($socket, 1024, 1024); +my $transport = $bufferedSocket; +my $protocol = new Thrift::BinaryProtocol($transport); +my $testClient = new ThriftTest::ThriftTestClient($protocol); + +eval{ +$transport->open(); +}; if($@){ + die(Dumper($@)); +} +my $start = gettimeofday(); + +# +# VOID TEST +# +print("testVoid()"); +$testClient->testVoid(); +print(" = void\n"); + +# +# STRING TEST +# +print("testString(\"Test\")"); +my $s = $testClient->testString("Test"); +print(" = \"$s\"\n"); + +# +# BYTE TEST +# +print("testByte(1)"); +my $u8 = $testClient->testByte(1); +print(" = $u8\n"); + +# +# I32 TEST +# +print("testI32(-1)"); +my $i32 = $testClient->testI32(-1); +print(" = $i32\n"); + +# +#I64 TEST +# +print("testI64(-34359738368)"); +my $i64 = $testClient->testI64(-34359738368); +print(" = $i64\n"); + +# +# DOUBLE TEST +# +print("testDouble(-852.234234234)"); +my $dub = $testClient->testDouble(-852.234234234); +print(" = $dub\n"); + +# +# STRUCT TEST +# +print("testStruct({\"Zero\", 1, -3, -5})"); +my $out = new ThriftTest::Xtruct(); +$out->string_thing("Zero"); +$out->byte_thing(1); +$out->i32_thing(-3); +$out->i64_thing(-5); +my $in = $testClient->testStruct($out); +print(" = {\"".$in->string_thing."\", ". + $in->byte_thing.", ". + $in->i32_thing.", ". + $in->i64_thing."}\n"); + +# +# NESTED STRUCT TEST +# +print("testNest({1, {\"Zero\", 1, -3, -5}, 5}"); +my $out2 = new ThriftTest::Xtruct2(); +$out2->byte_thing(1); +$out2->struct_thing($out); +$out2->i32_thing(5); +my $in2 = $testClient->testNest($out2); +$in = $in2->struct_thing; +print(" = {".$in2->byte_thing.", {\"". + $in->string_thing."\", ". + $in->byte_thing.", ". + $in->i32_thing.", ". + $in->i64_thing."}, ". + $in2->i32_thing."}\n"); + +# +# MAP TEST +# +my $mapout = {}; +for (my $i = 0; $i < 5; ++$i) { + $mapout->{$i} = $i-10; +} +print("testMap({"); +my $first = 1; +while( my($key,$val) = each %$mapout) { + if ($first) { + $first = 0; + } else { + print(", "); + } + print("$key => $val"); +} +print("})"); + + +my $mapin = $testClient->testMap($mapout); +print(" = {"); + +$first = 1; +while( my($key,$val) = each %$mapin){ + if ($first) { + $first = 0; + } else { + print(", "); + } + print("$key => $val"); +} +print("}\n"); + +# +# SET TEST +# +my $setout = []; +for (my $i = -2; $i < 3; ++$i) { + push(@$setout, $i); +} + +print("testSet({".join(",",@$setout)."})"); + +my $setin = $testClient->testSet($setout); + +print(" = {".join(",",@$setout)."}\n"); + +# +# LIST TEST +# +my $listout = []; +for (my $i = -2; $i < 3; ++$i) { + push(@$listout, $i); +} + +print("testList({".join(",",@$listout)."})"); + +my $listin = $testClient->testList($listout); + +print(" = {".join(",",@$listin)."}\n"); + +# +# ENUM TEST +# +print("testEnum(ONE)"); +my $ret = $testClient->testEnum(ThriftTest::Numberz::ONE); +print(" = $ret\n"); + +print("testEnum(TWO)"); +$ret = $testClient->testEnum(ThriftTest::Numberz::TWO); +print(" = $ret\n"); + +print("testEnum(THREE)"); +$ret = $testClient->testEnum(ThriftTest::Numberz::THREE); +print(" = $ret\n"); + +print("testEnum(FIVE)"); +$ret = $testClient->testEnum(ThriftTest::Numberz::FIVE); +print(" = $ret\n"); + +print("testEnum(EIGHT)"); +$ret = $testClient->testEnum(ThriftTest::Numberz::EIGHT); +print(" = $ret\n"); + +# +# TYPEDEF TEST +# +print("testTypedef(309858235082523)"); +my $uid = $testClient->testTypedef(309858235082523); +print(" = $uid\n"); + +# +# NESTED MAP TEST +# +print("testMapMap(1)"); +my $mm = $testClient->testMapMap(1); +print(" = {"); +while( my ($key,$val) = each %$mm) { + print("$key => {"); + while( my($k2,$v2) = each %$val) { + print("$k2 => $v2, "); + } + print("}, "); +} +print("}\n"); + +# +# INSANITY TEST +# +my $insane = new ThriftTest::Insanity(); +$insane->{userMap}->{ThriftTest::Numberz::FIVE} = 5000; +my $truck = new ThriftTest::Xtruct(); +$truck->string_thing("Truck"); +$truck->byte_thing(8); +$truck->i32_thing(8); +$truck->i64_thing(8); +push(@{$insane->{xtructs}}, $truck); + +print("testInsanity()"); +my $whoa = $testClient->testInsanity($insane); +print(" = {"); +while( my ($key,$val) = each %$whoa) { + print("$key => {"); + while( my($k2,$v2) = each %$val) { + print("$k2 => {"); + my $userMap = $v2->{userMap}; + print("{"); + if (ref($userMap) eq "HASH") { + while( my($k3,$v3) = each %$userMap) { + print("$k3 => $v3, "); + } + } + print("}, "); + + my $xtructs = $v2->{xtructs}; + print("{"); + if (ref($xtructs) eq "ARRAY") { + foreach my $x (@$xtructs) { + print("{\"".$x->{string_thing}."\", ". + $x->{byte_thing}.", ".$x->{i32_thing}.", ".$x->{i64_thing}."}, "); + } + } + print("}"); + + print("}, "); + } + print("}, "); +} +print("}\n"); + +# +# EXCEPTION TEST +# +print("testException('Xception')"); +eval { + $testClient->testException('Xception'); + print(" void\nFAILURE\n"); +}; if($@ && $@->UNIVERSAL::isa('ThriftTest::Xception')) { + print(' caught xception '.$@->{errorCode}.': '.$@->{message}."\n"); +} + + +# +# Normal tests done. +# +my $stop = gettimeofday(); +my $elp = sprintf("%d",1000*($stop - $start), 0); +print("Total time: $elp ms\n"); + +# +# Extraneous "I don't trust PHP to pack/unpack integer" tests +# + +# Max I32 +my $num = 2**30 + 2**30 - 1; +my $num2 = $testClient->testI32($num); +if ($num != $num2) { + print "Missed max32 $num = $num2\n"; +} + +# Min I32 +$num = 0 - 2**31; +$num2 = $testClient->testI32($num); +if ($num != $num2) { + print "Missed min32 $num = $num2\n"; +} + +# Max Number I can get out of my perl +$num = 2**40; +$num2 = $testClient->testI64($num); +if ($num != $num2) { + print "Missed max64 $num = $num2\n"; +} + +# Max Number I can get out of my perl +$num = 0 - 2**40; +$num2 = $testClient->testI64($num); +if ($num != $num2) { + print "Missed min64 $num = $num2\n"; +} + +$transport->close(); + + + diff --git a/test/php/Makefile b/test/php/Makefile new file mode 100644 index 00000000..aa35c6e9 --- /dev/null +++ b/test/php/Makefile @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Default target is everything +target: all + +# Tools +THRIFT = ../../compiler/cpp/thrift + +all: normal inline + +normal: stubs + +inline: stubs-inline + +stubs: ../ThriftTest.thrift + $(THRIFT) --gen php ../ThriftTest.thrift + +stubs-inline: ../ThriftTest.thrift + $(THRIFT) --gen php:inlined ../ThriftTest.thrift + +clean: + $(RM) -r gen-php gen-phpi diff --git a/test/php/TestClient.php b/test/php/TestClient.php new file mode 100644 index 00000000..6d640dac --- /dev/null +++ b/test/php/TestClient.php @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + 1) { + $host = $argv[0]; +} + +if ($argc > 2) { + $host = $argv[1]; +} + +$hosts = array('localhost'); + +$socket = new TSocket($host, $port); +$socket = new TSocketPool($hosts, $port); +$socket->setDebug(TRUE); + +if ($MODE == 'inline') { + $transport = $socket; + $testClient = new ThriftTestClient($transport); +} else { + $bufferedSocket = new TBufferedTransport($socket, 1024, 1024); + $transport = $bufferedSocket; + $protocol = new TBinaryProtocol($transport); + $testClient = new ThriftTestClient($protocol); +} + +$transport->open(); + +$start = microtime(true); + +/** + * VOID TEST + */ +print_r("testVoid()"); +$testClient->testVoid(); +print_r(" = void\n"); + +/** + * STRING TEST + */ +print_r("testString(\"Test\")"); +$s = $testClient->testString("Test"); +print_r(" = \"$s\"\n"); + +/** + * BYTE TEST + */ +print_r("testByte(1)"); +$u8 = $testClient->testByte(1); +print_r(" = $u8\n"); + +/** + * I32 TEST + */ +print_r("testI32(-1)"); +$i32 = $testClient->testI32(-1); +print_r(" = $i32\n"); + +/** + * I64 TEST + */ +print_r("testI64(-34359738368)"); +$i64 = $testClient->testI64(-34359738368); +print_r(" = $i64\n"); + +/** + * DOUBLE TEST + */ +print_r("testDouble(-852.234234234)"); +$dub = $testClient->testDouble(-852.234234234); +print_r(" = $dub\n"); + +/** + * STRUCT TEST + */ +print_r("testStruct({\"Zero\", 1, -3, -5})"); +$out = new Xtruct(); +$out->string_thing = "Zero"; +$out->byte_thing = 1; +$out->i32_thing = -3; +$out->i64_thing = -5; +$in = $testClient->testStruct($out); +print_r(" = {\"".$in->string_thing."\", ". + $in->byte_thing.", ". + $in->i32_thing.", ". + $in->i64_thing."}\n"); + +/** + * NESTED STRUCT TEST + */ +print_r("testNest({1, {\"Zero\", 1, -3, -5}), 5}"); +$out2 = new Xtruct2(); +$out2->byte_thing = 1; +$out2->struct_thing = $out; +$out2->i32_thing = 5; +$in2 = $testClient->testNest($out2); +$in = $in2->struct_thing; +print_r(" = {".$in2->byte_thing.", {\"". + $in->string_thing."\", ". + $in->byte_thing.", ". + $in->i32_thing.", ". + $in->i64_thing."}, ". + $in2->i32_thing."}\n"); + +/** + * MAP TEST + */ +$mapout = array(); +for ($i = 0; $i < 5; ++$i) { + $mapout[$i] = $i-10; +} +print_r("testMap({"); +$first = true; +foreach ($mapout as $key => $val) { + if ($first) { + $first = false; + } else { + print_r(", "); + } + print_r("$key => $val"); +} +print_r("})"); + +$mapin = $testClient->testMap($mapout); +print_r(" = {"); +$first = true; +foreach ($mapin as $key => $val) { + if ($first) { + $first = false; + } else { + print_r(", "); + } + print_r("$key => $val"); +} +print_r("}\n"); + +/** + * SET TEST + */ +$setout = array();; +for ($i = -2; $i < 3; ++$i) { + $setout []= $i; +} +print_r("testSet({"); +$first = true; +foreach ($setout as $val) { + if ($first) { + $first = false; + } else { + print_r(", "); + } + print_r($val); +} +print_r("})"); +$setin = $testClient->testSet($setout); +print_r(" = {"); +$first = true; +foreach ($setin as $val) { + if ($first) { + $first = false; + } else { + print_r(", "); + } + print_r($val); +} +print_r("}\n"); + +/** + * LIST TEST + */ +$listout = array(); +for ($i = -2; $i < 3; ++$i) { + $listout []= $i; +} +print_r("testList({"); +$first = true; +foreach ($listout as $val) { + if ($first) { + $first = false; + } else { + print_r(", "); + } + print_r($val); +} +print_r("})"); +$listin = $testClient->testList($listout); +print_r(" = {"); +$first = true; +foreach ($listin as $val) { + if ($first) { + $first = false; + } else { + print_r(", "); + } + print_r($val); +} +print_r("}\n"); + +/** + * ENUM TEST + */ +print_r("testEnum(ONE)"); +$ret = $testClient->testEnum(Numberz::ONE); +print_r(" = $ret\n"); + +print_r("testEnum(TWO)"); +$ret = $testClient->testEnum(Numberz::TWO); +print_r(" = $ret\n"); + +print_r("testEnum(THREE)"); +$ret = $testClient->testEnum(Numberz::THREE); +print_r(" = $ret\n"); + +print_r("testEnum(FIVE)"); +$ret = $testClient->testEnum(Numberz::FIVE); +print_r(" = $ret\n"); + +print_r("testEnum(EIGHT)"); +$ret = $testClient->testEnum(Numberz::EIGHT); +print_r(" = $ret\n"); + +/** + * TYPEDEF TEST + */ +print_r("testTypedef(309858235082523)"); +$uid = $testClient->testTypedef(309858235082523); +print_r(" = $uid\n"); + +/** + * NESTED MAP TEST + */ +print_r("testMapMap(1)"); +$mm = $testClient->testMapMap(1); +print_r(" = {"); +foreach ($mm as $key => $val) { + print_r("$key => {"); + foreach ($val as $k2 => $v2) { + print_r("$k2 => $v2, "); + } + print_r("}, "); +} +print_r("}\n"); + +/** + * INSANITY TEST + */ +$insane = new Insanity(); +$insane->userMap[Numberz::FIVE] = 5000; +$truck = new Xtruct(); +$truck->string_thing = "Truck"; +$truck->byte_thing = 8; +$truck->i32_thing = 8; +$truck->i64_thing = 8; +$insane->xtructs []= $truck; +print_r("testInsanity()"); +$whoa = $testClient->testInsanity($insane); +print_r(" = {"); +foreach ($whoa as $key => $val) { + print_r("$key => {"); + foreach ($val as $k2 => $v2) { + print_r("$k2 => {"); + $userMap = $v2->userMap; + print_r("{"); + if (is_array($usermap)) { + foreach ($userMap as $k3 => $v3) { + print_r("$k3 => $v3, "); + } + } + print_r("}, "); + + $xtructs = $v2->xtructs; + print_r("{"); + if (is_array($xtructs)) { + foreach ($xtructs as $x) { + print_r("{\"".$x->string_thing."\", ". + $x->byte_thing.", ".$x->i32_thing.", ".$x->i64_thing."}, "); + } + } + print_r("}"); + + print_r("}, "); + } + print_r("}, "); +} +print_r("}\n"); + +/** + * EXCEPTION TEST + */ +print_r("testException('Xception')"); +try { + $testClient->testException('Xception'); + print_r(" void\nFAILURE\n"); +} catch (Xception $x) { + print_r(' caught xception '.$x->errorCode.': '.$x->message."\n"); +} + + +/** + * Normal tests done. + */ + +$stop = microtime(true); +$elp = round(1000*($stop - $start), 0); +print_r("Total time: $elp ms\n"); + +/** + * Extraneous "I don't trust PHP to pack/unpack integer" tests + */ + +// Max I32 +$num = pow(2, 30) + (pow(2, 30) - 1); +$num2 = $testClient->testI32($num); +if ($num != $num2) { + print "Missed $num = $num2\n"; +} + +// Min I32 +$num = 0 - pow(2, 31); +$num2 = $testClient->testI32($num); +if ($num != $num2) { + print "Missed $num = $num2\n"; +} + +// Max I64 +$num = pow(2, 62) + (pow(2, 62) - 1); +$num2 = $testClient->testI64($num); +if ($num != $num2) { + print "Missed $num = $num2\n"; +} + +// Min I64 +$num = 0 - pow(2, 63); +$num2 = $testClient->testI64($num); +if ($num != $num2) { + print "Missed $num = $num2\n"; +} + +$transport->close(); +return; + +?> diff --git a/test/php/TestInline.php b/test/php/TestInline.php new file mode 100644 index 00000000..7066c461 --- /dev/null +++ b/test/php/TestInline.php @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + diff --git a/test/py/Makefile.am b/test/py/Makefile.am new file mode 100644 index 00000000..63b7a890 --- /dev/null +++ b/test/py/Makefile.am @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +THRIFT = $(top_srcdir)/compiler/cpp/thrift + +py_unit_tests = \ + SerializationTest.py \ + TestEof.py \ + TestSyntax.py \ + RunClientServer.py + +thrift_gen = \ + gen-py/ThriftTest/__init__.py \ + gen-py/DebugProtoTest/__init__.py + +helper_scripts= \ + TestClient.py \ + TestServer.py + +check_SCRIPTS= \ + $(thrift_gen) \ + $(py_unit_tests) \ + $(helper_scripts) + +TESTS= $(py_unit_tests) + + +gen-py/%/__init__.py: ../%.thrift + $(THRIFT) --gen py $< + +clean-local: + $(RM) -r gen-py diff --git a/test/py/RunClientServer.py b/test/py/RunClientServer.py new file mode 100755 index 00000000..2bd6094d --- /dev/null +++ b/test/py/RunClientServer.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import time +import subprocess +import sys +import os +import signal + +def relfile(fname): + return os.path.join(os.path.dirname(__file__), fname) + +FRAMED = ["TNonblockingServer"] + +def runTest(server_class): + print "Testing ", server_class + serverproc = subprocess.Popen([sys.executable, relfile("TestServer.py"), server_class]) + time.sleep(0.25) + try: + argv = [sys.executable, relfile("TestClient.py")] + if server_class in FRAMED: + argv.append('--framed') + if server_class == 'THttpServer': + argv.append('--http=/') + ret = subprocess.call(argv) + if ret != 0: + raise Exception("subprocess failed") + finally: + # fixme: should check that server didn't die + os.kill(serverproc.pid, signal.SIGKILL) + + # wait for shutdown + time.sleep(1) + +map(runTest, [ + "TSimpleServer", + "TThreadedServer", + "TThreadPoolServer", + "TForkingServer", + "TNonblockingServer", + "THttpServer", + ]) diff --git a/test/py/SerializationTest.py b/test/py/SerializationTest.py new file mode 100755 index 00000000..52bedd5e --- /dev/null +++ b/test/py/SerializationTest.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys, glob +sys.path.insert(0, './gen-py') +sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) + +from ThriftTest.ttypes import * +from thrift.transport import TTransport +from thrift.transport import TSocket +from thrift.protocol import TBinaryProtocol +import unittest +import time + +class AbstractTest(unittest.TestCase): + + def setUp(self): + self.v1obj = VersioningTestV1( + begin_in_both=12345, + old_string='aaa', + end_in_both=54321, + ) + + self.v2obj = VersioningTestV2( + begin_in_both=12345, + newint=1, + newbyte=2, + newshort=3, + newlong=4, + newdouble=5.0, + newstruct=Bonk(message="Hello!", type=123), + newlist=[7,8,9], + newset=[42,1,8], + newmap={1:2,2:3}, + newstring="Hola!", + end_in_both=54321, + ) + + def _serialize(self, obj): + trans = TTransport.TMemoryBuffer() + prot = self.protocol_factory.getProtocol(trans) + obj.write(prot) + return trans.getvalue() + + def _deserialize(self, objtype, data): + prot = self.protocol_factory.getProtocol(TTransport.TMemoryBuffer(data)) + ret = objtype() + ret.read(prot) + return ret + + def testForwards(self): + obj = self._deserialize(VersioningTestV2, self._serialize(self.v1obj)) + self.assertEquals(obj.begin_in_both, self.v1obj.begin_in_both) + self.assertEquals(obj.end_in_both, self.v1obj.end_in_both) + + def testBackwards(self): + obj = self._deserialize(VersioningTestV1, self._serialize(self.v2obj)) + self.assertEquals(obj.begin_in_both, self.v2obj.begin_in_both) + self.assertEquals(obj.end_in_both, self.v2obj.end_in_both) + + +class NormalBinaryTest(AbstractTest): + protocol_factory = TBinaryProtocol.TBinaryProtocolFactory() + +class AcceleratedBinaryTest(AbstractTest): + protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory() + + +class AcceleratedFramedTest(unittest.TestCase): + def testSplit(self): + """Test FramedTransport and BinaryProtocolAccelerated + + Tests that TBinaryProtocolAccelerated and TFramedTransport + play nicely together when a read spans a frame""" + + protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory() + bigstring = "".join(chr(byte) for byte in range(ord("a"), ord("z")+1)) + + databuf = TTransport.TMemoryBuffer() + prot = protocol_factory.getProtocol(databuf) + prot.writeI32(42) + prot.writeString(bigstring) + prot.writeI16(24) + data = databuf.getvalue() + cutpoint = len(data)/2 + parts = [ data[:cutpoint], data[cutpoint:] ] + + framed_buffer = TTransport.TMemoryBuffer() + framed_writer = TTransport.TFramedTransport(framed_buffer) + for part in parts: + framed_writer.write(part) + framed_writer.flush() + self.assertEquals(len(framed_buffer.getvalue()), len(data) + 8) + + # Recreate framed_buffer so we can read from it. + framed_buffer = TTransport.TMemoryBuffer(framed_buffer.getvalue()) + framed_reader = TTransport.TFramedTransport(framed_buffer) + prot = protocol_factory.getProtocol(framed_reader) + self.assertEqual(prot.readI32(), 42) + self.assertEqual(prot.readString(), bigstring) + self.assertEqual(prot.readI16(), 24) + + + +def suite(): + suite = unittest.TestSuite() + loader = unittest.TestLoader() + + suite.addTest(loader.loadTestsFromTestCase(NormalBinaryTest)) + suite.addTest(loader.loadTestsFromTestCase(AcceleratedBinaryTest)) + suite.addTest(loader.loadTestsFromTestCase(AcceleratedFramedTest)) + return suite + +if __name__ == "__main__": + unittest.main(defaultTest="suite", testRunner=unittest.TextTestRunner(verbosity=2)) diff --git a/test/py/TestClient.py b/test/py/TestClient.py new file mode 100755 index 00000000..64e5e872 --- /dev/null +++ b/test/py/TestClient.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys, glob +sys.path.insert(0, './gen-py') +sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) + +from ThriftTest import ThriftTest +from ThriftTest.ttypes import * +from thrift.transport import TTransport +from thrift.transport import TSocket +from thrift.transport import THttpClient +from thrift.protocol import TBinaryProtocol +import unittest +import time +from optparse import OptionParser + + +parser = OptionParser() +parser.set_defaults(framed=False, http_path=None, verbose=1, host='localhost', port=9090) +parser.add_option("--port", type="int", dest="port", + help="connect to server at port") +parser.add_option("--host", type="string", dest="host", + help="connect to server") +parser.add_option("--framed", action="store_true", dest="framed", + help="use framed transport") +parser.add_option("--http", dest="http_path", + help="Use the HTTP transport with the specified path") +parser.add_option('-v', '--verbose', action="store_const", + dest="verbose", const=2, + help="verbose output") +parser.add_option('-q', '--quiet', action="store_const", + dest="verbose", const=0, + help="minimal output") + +options, args = parser.parse_args() + +class AbstractTest(unittest.TestCase): + def setUp(self): + if options.http_path: + self.transport = THttpClient.THttpClient( + options.host, options.port, options.http_path) + else: + socket = TSocket.TSocket(options.host, options.port) + + # frame or buffer depending upon args + if options.framed: + self.transport = TTransport.TFramedTransport(socket) + else: + self.transport = TTransport.TBufferedTransport(socket) + + self.transport.open() + + protocol = self.protocol_factory.getProtocol(self.transport) + self.client = ThriftTest.Client(protocol) + + def tearDown(self): + # Close! + self.transport.close() + + def testVoid(self): + self.client.testVoid() + + def testString(self): + self.assertEqual(self.client.testString('Python'), 'Python') + + def testByte(self): + self.assertEqual(self.client.testByte(63), 63) + + def testI32(self): + self.assertEqual(self.client.testI32(-1), -1) + self.assertEqual(self.client.testI32(0), 0) + + def testI64(self): + self.assertEqual(self.client.testI64(-34359738368), -34359738368) + + def testDouble(self): + self.assertEqual(self.client.testDouble(-5.235098235), -5.235098235) + + def testStruct(self): + x = Xtruct() + x.string_thing = "Zero" + x.byte_thing = 1 + x.i32_thing = -3 + x.i64_thing = -5 + y = self.client.testStruct(x) + + self.assertEqual(y.string_thing, "Zero") + self.assertEqual(y.byte_thing, 1) + self.assertEqual(y.i32_thing, -3) + self.assertEqual(y.i64_thing, -5) + + def testException(self): + self.client.testException('Safe') + try: + self.client.testException('Xception') + self.fail("should have gotten exception") + except Xception, x: + self.assertEqual(x.errorCode, 1001) + self.assertEqual(x.message, 'Xception') + + try: + self.client.testException("throw_undeclared") + self.fail("should have thrown exception") + except Exception: # type is undefined + pass + + def testOneway(self): + start = time.time() + self.client.testOneway(0.5) + end = time.time() + self.assertTrue(end - start < 0.2, + "oneway sleep took %f sec" % (end - start)) + +class NormalBinaryTest(AbstractTest): + protocol_factory = TBinaryProtocol.TBinaryProtocolFactory() + +class AcceleratedBinaryTest(AbstractTest): + protocol_factory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory() + +def suite(): + suite = unittest.TestSuite() + loader = unittest.TestLoader() + + suite.addTest(loader.loadTestsFromTestCase(NormalBinaryTest)) + suite.addTest(loader.loadTestsFromTestCase(AcceleratedBinaryTest)) + return suite + +class OwnArgsTestProgram(unittest.TestProgram): + def parseArgs(self, argv): + if args: + self.testNames = args + else: + self.testNames = (self.defaultTest,) + self.createTests() + +if __name__ == "__main__": + OwnArgsTestProgram(defaultTest="suite", testRunner=unittest.TextTestRunner(verbosity=2)) diff --git a/test/py/TestEof.py b/test/py/TestEof.py new file mode 100755 index 00000000..7d64289d --- /dev/null +++ b/test/py/TestEof.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys, glob +sys.path.insert(0, './gen-py') +sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) + +from ThriftTest import ThriftTest +from ThriftTest.ttypes import * +from thrift.transport import TTransport +from thrift.transport import TSocket +from thrift.protocol import TBinaryProtocol +import unittest +import time + +class TestEof(unittest.TestCase): + + def setUp(self): + trans = TTransport.TMemoryBuffer() + prot = TBinaryProtocol.TBinaryProtocol(trans) + + x = Xtruct() + x.string_thing = "Zero" + x.byte_thing = 0 + + x.write(prot) + + x = Xtruct() + x.string_thing = "One" + x.byte_thing = 1 + + x.write(prot) + + self.data = trans.getvalue() + + def testTransportReadAll(self): + """Test that readAll on any type of transport throws an EOFError""" + trans = TTransport.TMemoryBuffer(self.data) + trans.readAll(1) + + try: + trans.readAll(10000) + except EOFError: + return + + self.fail("Should have gotten EOFError") + + def eofTestHelper(self, pfactory): + trans = TTransport.TMemoryBuffer(self.data) + prot = pfactory.getProtocol(trans) + + x = Xtruct() + x.read(prot) + self.assertEqual(x.string_thing, "Zero") + self.assertEqual(x.byte_thing, 0) + + x = Xtruct() + x.read(prot) + self.assertEqual(x.string_thing, "One") + self.assertEqual(x.byte_thing, 1) + + try: + x = Xtruct() + x.read(prot) + except EOFError: + return + + self.fail("Should have gotten EOFError") + + def eofTestHelperStress(self, pfactory): + """Teest the ability of TBinaryProtocol to deal with the removal of every byte in the file""" + # TODO: we should make sure this covers more of the code paths + + for i in xrange(0, len(self.data) + 1): + trans = TTransport.TMemoryBuffer(self.data[0:i]) + prot = pfactory.getProtocol(trans) + try: + x = Xtruct() + x.read(prot) + x.read(prot) + x.read(prot) + except EOFError: + continue + self.fail("Should have gotten an EOFError") + + def testBinaryProtocolEof(self): + """Test that TBinaryProtocol throws an EOFError when it reaches the end of the stream""" + self.eofTestHelper(TBinaryProtocol.TBinaryProtocolFactory()) + self.eofTestHelperStress(TBinaryProtocol.TBinaryProtocolFactory()) + + def testBinaryProtocolAcceleratedEof(self): + """Test that TBinaryProtocolAccelerated throws an EOFError when it reaches the end of the stream""" + self.eofTestHelper(TBinaryProtocol.TBinaryProtocolAcceleratedFactory()) + self.eofTestHelperStress(TBinaryProtocol.TBinaryProtocolAcceleratedFactory()) + +def suite(): + suite = unittest.TestSuite() + loader = unittest.TestLoader() + suite.addTest(loader.loadTestsFromTestCase(TestEof)) + return suite + +if __name__ == "__main__": + unittest.main(defaultTest="suite", testRunner=unittest.TextTestRunner(verbosity=2)) diff --git a/test/py/TestServer.py b/test/py/TestServer.py new file mode 100755 index 00000000..3d379eae --- /dev/null +++ b/test/py/TestServer.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys, glob, time +sys.path.insert(0, './gen-py') +sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) + +from ThriftTest import ThriftTest +from ThriftTest.ttypes import * +from thrift.transport import TTransport +from thrift.transport import TSocket +from thrift.protocol import TBinaryProtocol +from thrift.server import TServer, TNonblockingServer, THttpServer + +class TestHandler: + + def testVoid(self): + print 'testVoid()' + + def testString(self, str): + print 'testString(%s)' % str + return str + + def testByte(self, byte): + print 'testByte(%d)' % byte + return byte + + def testI16(self, i16): + print 'testI16(%d)' % i16 + return i16 + + def testI32(self, i32): + print 'testI32(%d)' % i32 + return i32 + + def testI64(self, i64): + print 'testI64(%d)' % i64 + return i64 + + def testDouble(self, dub): + print 'testDouble(%f)' % dub + return dub + + def testStruct(self, thing): + print 'testStruct({%s, %d, %d, %d})' % (thing.string_thing, thing.byte_thing, thing.i32_thing, thing.i64_thing) + return thing + + def testException(self, str): + print 'testException(%s)' % str + if str == 'Xception': + x = Xception() + x.errorCode = 1001 + x.message = str + raise x + elif str == "throw_undeclared": + raise ValueError("foo") + + def testOneway(self, seconds): + print 'testOneway(%d) => sleeping...' % seconds + time.sleep(seconds) + print 'done sleeping' + + def testNest(self, thing): + return thing + + def testMap(self, thing): + return thing + + def testSet(self, thing): + return thing + + def testList(self, thing): + return thing + + def testEnum(self, thing): + return thing + + def testTypedef(self, thing): + return thing + +pfactory = TBinaryProtocol.TBinaryProtocolFactory() +handler = TestHandler() +processor = ThriftTest.Processor(handler) + +if sys.argv[1] == "THttpServer": + server = THttpServer.THttpServer(processor, ('', 9090), pfactory) +else: + transport = TSocket.TServerSocket(9090) + tfactory = TTransport.TBufferedTransportFactory() + + if sys.argv[1] == "TNonblockingServer": + server = TNonblockingServer.TNonblockingServer(processor, transport) + else: + ServerClass = getattr(TServer, sys.argv[1]) + server = ServerClass(processor, transport, tfactory, pfactory) + +server.serve() diff --git a/test/py/TestSocket.py b/test/py/TestSocket.py new file mode 100755 index 00000000..2f7353fb --- /dev/null +++ b/test/py/TestSocket.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys, glob +sys.path.insert(0, './gen-py') +sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) + +from ThriftTest import ThriftTest +from ThriftTest.ttypes import * +from thrift.transport import TTransport +from thrift.transport import TSocket +from thrift.protocol import TBinaryProtocol +import unittest +import time +import socket +import random +from optparse import OptionParser + +class TimeoutTest(unittest.TestCase): + def setUp(self): + for i in xrange(50): + try: + # find a port we can use + self.listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = random.randint(10000, 30000) + self.listen_sock.bind(('localhost', self.port)) + self.listen_sock.listen(5) + break + except: + if i == 49: + raise + + def testConnectTimeout(self): + starttime = time.time() + + try: + leaky = [] + for i in xrange(100): + socket = TSocket.TSocket('localhost', self.port) + socket.setTimeout(10) + socket.open() + leaky.append(socket) + except: + self.assert_(time.time() - starttime < 5.0) + + def testWriteTimeout(self): + starttime = time.time() + + try: + socket = TSocket.TSocket('localhost', self.port) + socket.setTimeout(10) + socket.open() + lsock = self.listen_sock.accept() + while True: + socket.write("hi" * 100) + + except: + self.assert_(time.time() - starttime < 5.0) + +suite = unittest.TestSuite() +loader = unittest.TestLoader() + +suite.addTest(loader.loadTestsFromTestCase(TimeoutTest)) + +testRunner = unittest.TextTestRunner(verbosity=2) +testRunner.run(suite) diff --git a/test/py/TestSyntax.py b/test/py/TestSyntax.py new file mode 100755 index 00000000..df67d485 --- /dev/null +++ b/test/py/TestSyntax.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys, glob +sys.path.insert(0, './gen-py') +sys.path.insert(0, glob.glob('../../lib/py/build/lib.*')[0]) + +# Just import these generated files to make sure they are syntactically valid +from DebugProtoTest import EmptyService +from DebugProtoTest import Inherited diff --git a/test/py/explicit_module/runtest.sh b/test/py/explicit_module/runtest.sh new file mode 100755 index 00000000..2e5a4f1b --- /dev/null +++ b/test/py/explicit_module/runtest.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +rm -rf gen-py +../../../compiler/cpp/thrift --gen py test1.thrift || exit 1 +../../../compiler/cpp/thrift --gen py test2.thrift || exit 1 +PYTHONPATH=./gen-py python -c 'import foo.bar.baz' || exit 1 +PYTHONPATH=./gen-py python -c 'import test2' || exit 1 +PYTHONPATH=./gen-py python -c 'import test1' &>/dev/null && exit 1 # Should fail. +cp -r gen-py simple +../../../compiler/cpp/thrift -r --gen py test2.thrift || exit 1 +PYTHONPATH=./gen-py python -c 'import test2' || exit 1 +diff -ur simple gen-py > thediffs +file thediffs | grep -s -q empty || exit 1 +rm -rf simple thediffs +echo 'All tests pass!' diff --git a/test/py/explicit_module/test1.thrift b/test/py/explicit_module/test1.thrift new file mode 100644 index 00000000..ec600d7d --- /dev/null +++ b/test/py/explicit_module/test1.thrift @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace py foo.bar.baz + +struct astruct { + 1: i32 how_unoriginal; +} diff --git a/test/py/explicit_module/test2.thrift b/test/py/explicit_module/test2.thrift new file mode 100644 index 00000000..68f9da4d --- /dev/null +++ b/test/py/explicit_module/test2.thrift @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +include "test1.thrift" + +struct another { + 1: test1.astruct something; +} diff --git a/test/rb/Makefile.am b/test/rb/Makefile.am new file mode 100644 index 00000000..a6f431c8 --- /dev/null +++ b/test/rb/Makefile.am @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +THRIFT = $(top_srcdir)/compiler/cpp/thrift + +stubs: ../ThriftTest.thrift ../SmallTest.thrift + $(THRIFT) --gen rb ../ThriftTest.thrift + $(THRIFT) --gen rb ../SmallTest.thrift + +check: stubs + $(RUBY) test_suite.rb + diff --git a/test/rb/benchmarks/protocol_benchmark.rb b/test/rb/benchmarks/protocol_benchmark.rb new file mode 100644 index 00000000..05a8ee53 --- /dev/null +++ b/test/rb/benchmarks/protocol_benchmark.rb @@ -0,0 +1,174 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +$LOAD_PATH.unshift File.join(File.dirname(__FILE__), *%w[.. .. .. lib rb lib]) +$LOAD_PATH.unshift File.join(File.dirname(__FILE__), *%w[.. .. .. lib rb ext]) + +require 'thrift' + +require 'benchmark' +require 'rubygems' +require 'set' +require 'pp' + +# require 'ruby-debug' +# require 'ruby-prof' + +require File.join(File.dirname(__FILE__), '../fixtures/structs') + +transport1 = Thrift::MemoryBuffer.new +ruby_binary_protocol = Thrift::BinaryProtocol.new(transport1) + +transport2 = Thrift::MemoryBuffer.new +c_fast_binary_protocol = Thrift::BinaryProtocolAccelerated.new(transport2) + + +ooe = Fixtures::Structs::OneOfEach.new +ooe.im_true = true +ooe.im_false = false +ooe.a_bite = -42 +ooe.integer16 = 27000 +ooe.integer32 = 1<<24 +ooe.integer64 = 6000 * 1000 * 1000 +ooe.double_precision = Math::PI +ooe.some_characters = "Debug THIS!" +ooe.zomg_unicode = "\xd7\n\a\t" + +n1 = Fixtures::Structs::Nested1.new +n1.a_list = [] +n1.a_list << ooe << ooe << ooe << ooe +n1.i32_map = {} +n1.i32_map[1234] = ooe +n1.i32_map[46345] = ooe +n1.i32_map[-34264] = ooe +n1.i64_map = {} +n1.i64_map[43534986783945] = ooe +n1.i64_map[-32434639875122] = ooe +n1.dbl_map = {} +n1.dbl_map[324.65469834] = ooe +n1.dbl_map[-9458672340.4986798345112] = ooe +n1.str_map = {} +n1.str_map['sdoperuix'] = ooe +n1.str_map['pwoerxclmn'] = ooe + +n2 = Fixtures::Structs::Nested2.new +n2.a_list = [] +n2.a_list << n1 << n1 << n1 << n1 << n1 +n2.i32_map = {} +n2.i32_map[398345] = n1 +n2.i32_map[-2345] = n1 +n2.i32_map[12312] = n1 +n2.i64_map = {} +n2.i64_map[2349843765934] = n1 +n2.i64_map[-123234985495] = n1 +n2.i64_map[0] = n1 +n2.dbl_map = {} +n2.dbl_map[23345345.38927834] = n1 +n2.dbl_map[-1232349.5489345] = n1 +n2.dbl_map[-234984574.23498725] = n1 +n2.str_map = {} +n2.str_map[''] = n1 +n2.str_map['sdflkertpioux'] = n1 +n2.str_map['sdfwepwdcjpoi'] = n1 + +n3 = Fixtures::Structs::Nested3.new +n3.a_list = [] +n3.a_list << n2 << n2 << n2 << n2 << n2 +n3.i32_map = {} +n3.i32_map[398345] = n2 +n3.i32_map[-2345] = n2 +n3.i32_map[12312] = n2 +n3.i64_map = {} +n3.i64_map[2349843765934] = n2 +n3.i64_map[-123234985495] = n2 +n3.i64_map[0] = n2 +n3.dbl_map = {} +n3.dbl_map[23345345.38927834] = n2 +n3.dbl_map[-1232349.5489345] = n2 +n3.dbl_map[-234984574.23498725] = n2 +n3.str_map = {} +n3.str_map[''] = n2 +n3.str_map['sdflkertpioux'] = n2 +n3.str_map['sdfwepwdcjpoi'] = n2 + +n4 = Fixtures::Structs::Nested4.new +n4.a_list = [] +n4.a_list << n3 +n4.i32_map = {} +n4.i32_map[-2345] = n3 +n4.i64_map = {} +n4.i64_map[2349843765934] = n3 +n4.dbl_map = {} +n4.dbl_map[-1232349.5489345] = n3 +n4.str_map = {} +n4.str_map[''] = n3 + + +# prof = RubyProf.profile do +# n4.write(c_fast_binary_protocol) +# Fixtures::Structs::Nested4.new.read(c_fast_binary_protocol) +# end +# +# printer = RubyProf::GraphHtmlPrinter.new(prof) +# printer.print(STDOUT, :min_percent=>0) + +Benchmark.bmbm do |x| + x.report("ruby write large (1MB) structure once") do + n4.write(ruby_binary_protocol) + end + + x.report("ruby read large (1MB) structure once") do + Fixtures::Structs::Nested4.new.read(ruby_binary_protocol) + end + + x.report("c write large (1MB) structure once") do + n4.write(c_fast_binary_protocol) + end + + x.report("c read large (1MB) structure once") do + Fixtures::Structs::Nested4.new.read(c_fast_binary_protocol) + end + + + + x.report("ruby write 10_000 small structures") do + 10_000.times do + ooe.write(ruby_binary_protocol) + end + end + + x.report("ruby read 10_000 small structures") do + 10_000.times do + Fixtures::Structs::OneOfEach.new.read(ruby_binary_protocol) + end + end + + x.report("c write 10_000 small structures") do + 10_000.times do + ooe.write(c_fast_binary_protocol) + end + end + + x.report("c read 10_000 small structures") do + 10_000.times do + Fixtures::Structs::OneOfEach.new.read(c_fast_binary_protocol) + end + end + +end diff --git a/test/rb/core/test_backwards_compatability.rb b/test/rb/core/test_backwards_compatability.rb new file mode 100644 index 00000000..0577515d --- /dev/null +++ b/test/rb/core/test_backwards_compatability.rb @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.join(File.dirname(__FILE__), '../test_helper') + +require 'thrift' + +class TestThriftException < Test::Unit::TestCase + def test_has_accessible_message + msg = "hi there thrift" + assert_equal msg, Thrift::Exception.new(msg).message + end +end + diff --git a/test/rb/core/test_exceptions.rb b/test/rb/core/test_exceptions.rb new file mode 100644 index 00000000..f41587a7 --- /dev/null +++ b/test/rb/core/test_exceptions.rb @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.join(File.dirname(__FILE__), '../test_helper') + +require 'thrift' + +class TestException < Test::Unit::TestCase + def test_has_accessible_message + msg = "hi there thrift" + assert_equal msg, Thrift::Exception.new(msg).message + end +end + diff --git a/test/rb/core/transport/test_transport.rb b/test/rb/core/transport/test_transport.rb new file mode 100644 index 00000000..52755c1d --- /dev/null +++ b/test/rb/core/transport/test_transport.rb @@ -0,0 +1,70 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.join(File.dirname(__FILE__), '../../test_helper') + +require 'thrift' + +class DummyTransport < Thrift::BaseTransport + def initialize(data) + @data = data + end + + def read(size) + @data.slice!(0, size) + end +end + +# TTransport is basically an abstract class, but isn't raising NotImplementedError +class TestThriftTransport < Test::Unit::TestCase + def setup + @trans = Thrift::BaseTransport.new + end + + def test_open? + assert_nil @trans.open? + end + + def test_open + assert_nil @trans.open + end + + def test_close + assert_nil @trans.close + end + + # TODO: + # This doesn't necessarily test he right thing. + # It _looks_ like read isn't guarenteed to return the length + # you ask for and read_all is. This means our test needs to check + # for blocking. -- Kevin Clark 3/27/08 + def test_read_all + # Implements read + t = DummyTransport.new("hello") + assert_equal "hello", t.read_all(5) + end + + def test_write + assert_nil @trans.write(5) # arbitrary value + end + + def test_flush + assert_nil @trans.flush + end +end diff --git a/test/rb/fixtures/structs.rb b/test/rb/fixtures/structs.rb new file mode 100644 index 00000000..ebbeb0a7 --- /dev/null +++ b/test/rb/fixtures/structs.rb @@ -0,0 +1,298 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require 'thrift' + +module Fixtures + module Structs + class OneBool + include Thrift::Struct + attr_accessor :bool + FIELDS = { + 1 => {:type => Thrift::Types::BOOL, :name => 'bool'} + } + + def validate + end + end + + class OneByte + include Thrift::Struct + attr_accessor :byte + FIELDS = { + 1 => {:type => Thrift::Types::BYTE, :name => 'byte'} + } + + def validate + end + end + + class OneI16 + include Thrift::Struct + attr_accessor :i16 + FIELDS = { + 1 => {:type => Thrift::Types::I16, :name => 'i16'} + } + + def validate + end + end + + class OneI32 + include Thrift::Struct + attr_accessor :i32 + FIELDS = { + 1 => {:type => Thrift::Types::I32, :name => 'i32'} + } + + def validate + end + end + + class OneI64 + include Thrift::Struct + attr_accessor :i64 + FIELDS = { + 1 => {:type => Thrift::Types::I64, :name => 'i64'} + } + + def validate + end + end + + class OneDouble + include Thrift::Struct + attr_accessor :double + FIELDS = { + 1 => {:type => Thrift::Types::DOUBLE, :name => 'double'} + } + + def validate + end + end + + class OneString + include Thrift::Struct + attr_accessor :string + FIELDS = { + 1 => {:type => Thrift::Types::STRING, :name => 'string'} + } + + def validate + end + end + + class OneMap + include Thrift::Struct + attr_accessor :map + FIELDS = { + 1 => {:type => Thrift::Types::MAP, :name => 'map', :key => {:type => Thrift::Types::STRING}, :value => {:type => Thrift::Types::STRING}} + } + + def validate + end + end + + class NestedMap + include Thrift::Struct + attr_accessor :map + FIELDS = { + 0 => {:type => Thrift::Types::MAP, :name => 'map', :key => {:type => Thrift::Types::I32}, :value => {:type => Thrift::Types::MAP, :key => {:type => Thrift::Types::I32}, :value => {:type => Thrift::Types::I32}}} + } + + def validate + end + end + + class OneList + include Thrift::Struct + attr_accessor :list + FIELDS = { + 1 => {:type => Thrift::Types::LIST, :name => 'list', :element => {:type => Thrift::Types::STRING}} + } + + def validate + end + end + + class NestedList + include Thrift::Struct + attr_accessor :list + FIELDS = { + 0 => {:type => Thrift::Types::LIST, :name => 'list', :element => {:type => Thrift::Types::LIST, :element => { :type => Thrift::Types::I32 } } } + } + + def validate + end + end + + class OneSet + include Thrift::Struct + attr_accessor :set + FIELDS = { + 1 => {:type => Thrift::Types::SET, :name => 'set', :element => {:type => Thrift::Types::STRING}} + } + + def validate + end + end + + class NestedSet + include Thrift::Struct + attr_accessor :set + FIELDS = { + 1 => {:type => Thrift::Types::SET, :name => 'set', :element => {:type => Thrift::Types::SET, :element => { :type => Thrift::Types::STRING } }} + } + + def validate + end + end + + # struct OneOfEach { + # 1: bool im_true, + # 2: bool im_false, + # 3: byte a_bite, + # 4: i16 integer16, + # 5: i32 integer32, + # 6: i64 integer64, + # 7: double double_precision, + # 8: string some_characters, + # 9: string zomg_unicode, + # 10: bool what_who, + # 11: binary base64, + # } + class OneOfEach + include Thrift::Struct + attr_accessor :im_true, :im_false, :a_bite, :integer16, :integer32, :integer64, :double_precision, :some_characters, :zomg_unicode, :what_who, :base64 + FIELDS = { + 1 => {:type => Thrift::Types::BOOL, :name => 'im_true'}, + 2 => {:type => Thrift::Types::BOOL, :name => 'im_false'}, + 3 => {:type => Thrift::Types::BYTE, :name => 'a_bite'}, + 4 => {:type => Thrift::Types::I16, :name => 'integer16'}, + 5 => {:type => Thrift::Types::I32, :name => 'integer32'}, + 6 => {:type => Thrift::Types::I64, :name => 'integer64'}, + 7 => {:type => Thrift::Types::DOUBLE, :name => 'double_precision'}, + 8 => {:type => Thrift::Types::STRING, :name => 'some_characters'}, + 9 => {:type => Thrift::Types::STRING, :name => 'zomg_unicode'}, + 10 => {:type => Thrift::Types::BOOL, :name => 'what_who'}, + 11 => {:type => Thrift::Types::STRING, :name => 'base64'} + } + + # Added for assert_equal + def ==(other) + [:im_true, :im_false, :a_bite, :integer16, :integer32, :integer64, :double_precision, :some_characters, :zomg_unicode, :what_who, :base64].each do |f| + var = "@#{f}" + return false if instance_variable_get(var) != other.instance_variable_get(var) + end + true + end + + def validate + end + end + + # struct Nested1 { + # 1: list a_list + # 2: map i32_map + # 3: map i64_map + # 4: map dbl_map + # 5: map str_map + # } + class Nested1 + include Thrift::Struct + attr_accessor :a_list, :i32_map, :i64_map, :dbl_map, :str_map + FIELDS = { + 1 => {:type => Thrift::Types::LIST, :name => 'a_list', :element => {:type => Thrift::Types::STRUCT, :class => OneOfEach}}, + 2 => {:type => Thrift::Types::MAP, :name => 'i32_map', :key => {:type => Thrift::Types::I32}, :value => {:type => Thrift::Types::STRUCT, :class => OneOfEach}}, + 3 => {:type => Thrift::Types::MAP, :name => 'i64_map', :key => {:type => Thrift::Types::I64}, :value => {:type => Thrift::Types::STRUCT, :class => OneOfEach}}, + 4 => {:type => Thrift::Types::MAP, :name => 'dbl_map', :key => {:type => Thrift::Types::DOUBLE}, :value => {:type => Thrift::Types::STRUCT, :class => OneOfEach}}, + 5 => {:type => Thrift::Types::MAP, :name => 'str_map', :key => {:type => Thrift::Types::STRING}, :value => {:type => Thrift::Types::STRUCT, :class => OneOfEach}} + } + + def validate + end + end + + # struct Nested2 { + # 1: list a_list + # 2: map i32_map + # 3: map i64_map + # 4: map dbl_map + # 5: map str_map + # } + class Nested2 + include Thrift::Struct + attr_accessor :a_list, :i32_map, :i64_map, :dbl_map, :str_map + FIELDS = { + 1 => {:type => Thrift::Types::LIST, :name => 'a_list', :element => {:type => Thrift::Types::STRUCT, :class => Nested1}}, + 2 => {:type => Thrift::Types::MAP, :name => 'i32_map', :key => {:type => Thrift::Types::I32}, :value => {:type => Thrift::Types::STRUCT, :class => Nested1}}, + 3 => {:type => Thrift::Types::MAP, :name => 'i64_map', :key => {:type => Thrift::Types::I64}, :value => {:type => Thrift::Types::STRUCT, :class => Nested1}}, + 4 => {:type => Thrift::Types::MAP, :name => 'dbl_map', :key => {:type => Thrift::Types::DOUBLE}, :value => {:type => Thrift::Types::STRUCT, :class => Nested1}}, + 5 => {:type => Thrift::Types::MAP, :name => 'str_map', :key => {:type => Thrift::Types::STRING}, :value => {:type => Thrift::Types::STRUCT, :class => Nested1}} + } + + def validate + end + end + + # struct Nested3 { + # 1: list a_list + # 2: map i32_map + # 3: map i64_map + # 4: map dbl_map + # 5: map str_map + # } + class Nested3 + include Thrift::Struct + attr_accessor :a_list, :i32_map, :i64_map, :dbl_map, :str_map + FIELDS = { + 1 => {:type => Thrift::Types::LIST, :name => 'a_list', :element => {:type => Thrift::Types::STRUCT, :class => Nested2}}, + 2 => {:type => Thrift::Types::MAP, :name => 'i32_map', :key => {:type => Thrift::Types::I32}, :value => {:type => Thrift::Types::STRUCT, :class => Nested2}}, + 3 => {:type => Thrift::Types::MAP, :name => 'i64_map', :key => {:type => Thrift::Types::I64}, :value => {:type => Thrift::Types::STRUCT, :class => Nested2}}, + 4 => {:type => Thrift::Types::MAP, :name => 'dbl_map', :key => {:type => Thrift::Types::DOUBLE}, :value => {:type => Thrift::Types::STRUCT, :class => Nested2}}, + 5 => {:type => Thrift::Types::MAP, :name => 'str_map', :key => {:type => Thrift::Types::STRING}, :value => {:type => Thrift::Types::STRUCT, :class => Nested2}} + } + + def validate + end + end + + # struct Nested4 { + # 1: list a_list + # 2: map i32_map + # 3: map i64_map + # 4: map dbl_map + # 5: map str_map + # } + class Nested4 + include Thrift::Struct + attr_accessor :a_list, :i32_map, :i64_map, :dbl_map, :str_map + FIELDS = { + 1 => {:type => Thrift::Types::LIST, :name => 'a_list', :element => {:type => Thrift::Types::STRUCT, :class => Nested3}}, + 2 => {:type => Thrift::Types::MAP, :name => 'i32_map', :key => {:type => Thrift::Types::I32}, :value => {:type => Thrift::Types::STRUCT, :class => Nested3}}, + 3 => {:type => Thrift::Types::MAP, :name => 'i64_map', :key => {:type => Thrift::Types::I64}, :value => {:type => Thrift::Types::STRUCT, :class => Nested3}}, + 4 => {:type => Thrift::Types::MAP, :name => 'dbl_map', :key => {:type => Thrift::Types::DOUBLE}, :value => {:type => Thrift::Types::STRUCT, :class => Nested3}}, + 5 => {:type => Thrift::Types::MAP, :name => 'str_map', :key => {:type => Thrift::Types::STRING}, :value => {:type => Thrift::Types::STRUCT, :class => Nested3}} + } + + def validate + end + end + end +end diff --git a/test/rb/generation/test_enum.rb b/test/rb/generation/test_enum.rb new file mode 100644 index 00000000..7d3f08ba --- /dev/null +++ b/test/rb/generation/test_enum.rb @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.join(File.dirname(__FILE__), '../test_helper') +require 'thrift_test' + +class TestEnumGeneration < Test::Unit::TestCase + include Thrift::Test + def test_enum_valid_values + assert_equal(Numberz::VALID_VALUES, Set.new([Numberz::ONE, Numberz::TWO, Numberz::THREE, Numberz::FIVE, Numberz::SIX, Numberz::EIGHT])) + end +end \ No newline at end of file diff --git a/test/rb/generation/test_struct.rb b/test/rb/generation/test_struct.rb new file mode 100644 index 00000000..3bd4fc9b --- /dev/null +++ b/test/rb/generation/test_struct.rb @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.join(File.dirname(__FILE__), '../test_helper') +require 'small_service' + +class TestStructGeneration < Test::Unit::TestCase + + def test_default_values + hello = TestNamespace::Hello.new + + assert_kind_of(TestNamespace::Hello, hello) + assert_nil(hello.complexer) + + assert_equal(hello.simple, 53) + assert_equal(hello.words, 'words') + + assert_kind_of(TestNamespace::Goodbyez, hello.thinz) + assert_equal(hello.thinz.val, 36632) + + assert_kind_of(Hash, hello.complex) + assert_equal(hello.complex, { 6243 => 632, 2355 => 532, 23 => 532}) + + bool_passer = TestNamespace::BoolPasser.new(:value => false) + assert_equal false, bool_passer.value + end + + def test_goodbyez + assert_equal(TestNamespace::Goodbyez.new.val, 325) + end + +end diff --git a/test/rb/integration/accelerated_buffered_client.rb b/test/rb/integration/accelerated_buffered_client.rb new file mode 100644 index 00000000..7cec1df5 --- /dev/null +++ b/test/rb/integration/accelerated_buffered_client.rb @@ -0,0 +1,163 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.join(File.dirname(__FILE__), '../test_helper') + +require 'thrift' +require 'ThriftTest' + +class AcceleratedBufferedClientTest < Test::Unit::TestCase + def setup + unless @socket + @socket = Thrift::Socket.new('localhost', 9090) + @protocol = Thrift::BinaryProtocolAccelerated.new(Thrift::BufferedTransport.new(@socket)) + @client = Thrift::Test::ThriftTest::Client.new(@protocol) + @socket.open + end + end + + def test_string + assert_equal(@client.testString('string'), 'string') + end + + def test_byte + val = 8 + assert_equal(@client.testByte(val), val) + assert_equal(@client.testByte(-val), -val) + end + + def test_i32 + val = 32 + assert_equal(@client.testI32(val), val) + assert_equal(@client.testI32(-val), -val) + end + + def test_i64 + val = 64 + assert_equal(@client.testI64(val), val) + assert_equal(@client.testI64(-val), -val) + end + + def test_double + val = 3.14 + assert_equal(@client.testDouble(val), val) + assert_equal(@client.testDouble(-val), -val) + assert_kind_of(Float, @client.testDouble(val)) + end + + def test_map + val = {1 => 1, 2 => 2, 3 => 3} + assert_equal(@client.testMap(val), val) + assert_kind_of(Hash, @client.testMap(val)) + end + + def test_list + val = [1,2,3,4,5] + assert_equal(@client.testList(val), val) + assert_kind_of(Array, @client.testList(val)) + end + + def test_enum + val = Thrift::Test::Numberz::SIX + ret = @client.testEnum(val) + + assert_equal(ret, 6) + assert_kind_of(Fixnum, ret) + end + + def test_typedef + #UserId testTypedef(1: UserId thing), + true + end + + def test_set + val = Set.new([1,2,3]) + assert_equal(@client.testSet(val), val) + assert_kind_of(Set, @client.testSet(val)) + end + + def get_struct + Thrift::Test::Xtruct.new({'string_thing' => 'hi!', 'i32_thing' => 4 }) + end + + def test_struct + ret = @client.testStruct(get_struct) + + assert_nil(ret.byte_thing, nil) + assert_nil(ret.i64_thing, nil) + assert_equal(ret.string_thing, 'hi!') + assert_equal(ret.i32_thing, 4) + assert_kind_of(Thrift::Test::Xtruct, ret) + end + + def test_nest + struct2 = Thrift::Test::Xtruct2.new({'struct_thing' => get_struct, 'i32_thing' => 10}) + + ret = @client.testNest(struct2) + + assert_nil(ret.struct_thing.byte_thing, nil) + assert_nil(ret.struct_thing.i64_thing, nil) + assert_equal(ret.struct_thing.string_thing, 'hi!') + assert_equal(ret.struct_thing.i32_thing, 4) + assert_equal(ret.i32_thing, 10) + + assert_kind_of(Thrift::Test::Xtruct, ret.struct_thing) + assert_kind_of(Thrift::Test::Xtruct2, ret) + end + + def test_insane + insane = Thrift::Test::Insanity.new({ + 'userMap' => { Thrift::Test::Numberz::ONE => 44 }, + 'xtructs' => [get_struct, + Thrift::Test::Xtruct.new({ + 'string_thing' => 'hi again', + 'i32_thing' => 12 + }) + ] + }) + + ret = @client.testInsanity(insane) + + assert_not_nil(ret[44]) + assert_not_nil(ret[44][1]) + + struct = ret[44][1] + + assert_equal(struct.userMap[Thrift::Test::Numberz::ONE], 44) + assert_equal(struct.xtructs[1].string_thing, 'hi again') + assert_equal(struct.xtructs[1].i32_thing, 12) + + assert_kind_of(Hash, struct.userMap) + assert_kind_of(Array, struct.xtructs) + assert_kind_of(Thrift::Test::Insanity, struct) + end + + def test_map_map + ret = @client.testMapMap(4) + assert_kind_of(Hash, ret) + assert_equal(ret, { 4 => { 4 => 4}}) + end + + def test_exception + assert_raise Thrift::Test::Xception do + @client.testException('foo') + end + end +end + diff --git a/test/rb/integration/accelerated_buffered_server.rb b/test/rb/integration/accelerated_buffered_server.rb new file mode 100644 index 00000000..1ca66e54 --- /dev/null +++ b/test/rb/integration/accelerated_buffered_server.rb @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +$:.push File.dirname(__FILE__) + '/../gen-rb' +$:.push File.join(File.dirname(__FILE__), '../../../lib/rb/lib') +$:.push File.join(File.dirname(__FILE__), '../../../lib/rb/ext') + +require 'thrift' +require 'ThriftTest' + +class SimpleHandler + [:testString, :testByte, :testI32, :testI64, :testDouble, + :testStruct, :testMap, :testSet, :testList, :testNest, + :testEnum, :testTypedef].each do |meth| + + define_method(meth) do |thing| + thing + end + + end + + def testInsanity(thing) + num, uid = thing.userMap.find { true } + return {uid => {num => thing}} + end + + def testMapMap(thing) + return {thing => {thing => thing}} + end + + def testEnum(thing) + return thing + end + + def testTypedef(thing) + return thing + end + + def testException(thing) + raise Thrift::Test::Xception, :message => 'error' + end +end + +@handler = SimpleHandler.new +@processor = Thrift::Test::ThriftTest::Processor.new(@handler) +@transport = Thrift::ServerSocket.new(9090) +@server = Thrift::ThreadedServer.new(@processor, @transport, Thrift::BufferedTransportFactory.new, Thrift::BinaryProtocolAcceleratedFactory.new) + +@server.serve diff --git a/test/rb/integration/buffered_client.rb b/test/rb/integration/buffered_client.rb new file mode 100644 index 00000000..1a925ccf --- /dev/null +++ b/test/rb/integration/buffered_client.rb @@ -0,0 +1,163 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.join(File.dirname(__FILE__), '../test_helper') + +require 'thrift' +require 'ThriftTest' + +class BufferedClientTest < Test::Unit::TestCase + def setup + unless @socket + @socket = Thrift::Socket.new('localhost', 9090) + @protocol = Thrift::BinaryProtocol.new(Thrift::BufferedTransport.new(@socket)) + @client = Thrift::Test::ThriftTest::Client.new(@protocol) + @socket.open + end + end + + def test_string + assert_equal(@client.testString('string'), 'string') + end + + def test_byte + val = 8 + assert_equal(@client.testByte(val), val) + assert_equal(@client.testByte(-val), -val) + end + + def test_i32 + val = 32 + assert_equal(@client.testI32(val), val) + assert_equal(@client.testI32(-val), -val) + end + + def test_i64 + val = 64 + assert_equal(@client.testI64(val), val) + assert_equal(@client.testI64(-val), -val) + end + + def test_double + val = 3.14 + assert_equal(@client.testDouble(val), val) + assert_equal(@client.testDouble(-val), -val) + assert_kind_of(Float, @client.testDouble(val)) + end + + def test_map + val = {1 => 1, 2 => 2, 3 => 3} + assert_equal(@client.testMap(val), val) + assert_kind_of(Hash, @client.testMap(val)) + end + + def test_list + val = [1,2,3,4,5] + assert_equal(@client.testList(val), val) + assert_kind_of(Array, @client.testList(val)) + end + + def test_enum + val = Thrift::Test::Numberz::SIX + ret = @client.testEnum(val) + + assert_equal(ret, 6) + assert_kind_of(Fixnum, ret) + end + + def test_typedef + #UserId testTypedef(1: UserId thing), + true + end + + def test_set + val = Set.new([1,2,3]) + assert_equal(@client.testSet(val), val) + assert_kind_of(Set, @client.testSet(val)) + end + + def get_struct + Thrift::Test::Xtruct.new({'string_thing' => 'hi!', 'i32_thing' => 4 }) + end + + def test_struct + ret = @client.testStruct(get_struct) + + assert_nil(ret.byte_thing, nil) + assert_nil(ret.i64_thing, nil) + assert_equal(ret.string_thing, 'hi!') + assert_equal(ret.i32_thing, 4) + assert_kind_of(Thrift::Test::Xtruct, ret) + end + + def test_nest + struct2 = Thrift::Test::Xtruct2.new({'struct_thing' => get_struct, 'i32_thing' => 10}) + + ret = @client.testNest(struct2) + + assert_nil(ret.struct_thing.byte_thing, nil) + assert_nil(ret.struct_thing.i64_thing, nil) + assert_equal(ret.struct_thing.string_thing, 'hi!') + assert_equal(ret.struct_thing.i32_thing, 4) + assert_equal(ret.i32_thing, 10) + + assert_kind_of(Thrift::Test::Xtruct, ret.struct_thing) + assert_kind_of(Thrift::Test::Xtruct2, ret) + end + + def test_insane + insane = Thrift::Test::Insanity.new({ + 'userMap' => { Thrift::Test::Numberz::ONE => 44 }, + 'xtructs' => [get_struct, + Thrift::Test::Xtruct.new({ + 'string_thing' => 'hi again', + 'i32_thing' => 12 + }) + ] + }) + + ret = @client.testInsanity(insane) + + assert_not_nil(ret[44]) + assert_not_nil(ret[44][1]) + + struct = ret[44][1] + + assert_equal(struct.userMap[Thrift::Test::Numberz::ONE], 44) + assert_equal(struct.xtructs[1].string_thing, 'hi again') + assert_equal(struct.xtructs[1].i32_thing, 12) + + assert_kind_of(Hash, struct.userMap) + assert_kind_of(Array, struct.xtructs) + assert_kind_of(Thrift::Test::Insanity, struct) + end + + def test_map_map + ret = @client.testMapMap(4) + assert_kind_of(Hash, ret) + assert_equal(ret, { 4 => { 4 => 4}}) + end + + def test_exception + assert_raise Thrift::Test::Xception do + @client.testException('foo') + end + end +end + diff --git a/test/rb/integration/simple_client.rb b/test/rb/integration/simple_client.rb new file mode 100644 index 00000000..1064822a --- /dev/null +++ b/test/rb/integration/simple_client.rb @@ -0,0 +1,163 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.join(File.dirname(__FILE__), '../test_helper') + +require 'thrift' +require 'ThriftTest' + +class SimpleClientTest < Test::Unit::TestCase + def setup + unless @socket + @socket = Thrift::Socket.new('localhost', 9090) + @protocol = Thrift::BinaryProtocol.new(@socket) + @client = Thrift::Test::ThriftTest::Client.new(@protocol) + @socket.open + end + end + + def test_string + assert_equal(@client.testString('string'), 'string') + end + + def test_byte + val = 8 + assert_equal(@client.testByte(val), val) + assert_equal(@client.testByte(-val), -val) + end + + def test_i32 + val = 32 + assert_equal(@client.testI32(val), val) + assert_equal(@client.testI32(-val), -val) + end + + def test_i64 + val = 64 + assert_equal(@client.testI64(val), val) + assert_equal(@client.testI64(-val), -val) + end + + def test_double + val = 3.14 + assert_equal(@client.testDouble(val), val) + assert_equal(@client.testDouble(-val), -val) + assert_kind_of(Float, @client.testDouble(val)) + end + + def test_map + val = {1 => 1, 2 => 2, 3 => 3} + assert_equal(@client.testMap(val), val) + assert_kind_of(Hash, @client.testMap(val)) + end + + def test_list + val = [1,2,3,4,5] + assert_equal(@client.testList(val), val) + assert_kind_of(Array, @client.testList(val)) + end + + def test_enum + val = Thrift::Test::Numberz::SIX + ret = @client.testEnum(val) + + assert_equal(ret, 6) + assert_kind_of(Fixnum, ret) + end + + def test_typedef + #UserId testTypedef(1: UserId thing), + true + end + + def test_set + val = Set.new([1,2,3]) + assert_equal(@client.testSet(val), val) + assert_kind_of(Set, @client.testSet(val)) + end + + def get_struct + Thrift::Test::Xtruct.new({'string_thing' => 'hi!', 'i32_thing' => 4 }) + end + + def test_struct + ret = @client.testStruct(get_struct) + + assert_nil(ret.byte_thing, nil) + assert_nil(ret.i64_thing, nil) + assert_equal(ret.string_thing, 'hi!') + assert_equal(ret.i32_thing, 4) + assert_kind_of(Thrift::Test::Xtruct, ret) + end + + def test_nest + struct2 = Thrift::Test::Xtruct2.new({'struct_thing' => get_struct, 'i32_thing' => 10}) + + ret = @client.testNest(struct2) + + assert_nil(ret.struct_thing.byte_thing, nil) + assert_nil(ret.struct_thing.i64_thing, nil) + assert_equal(ret.struct_thing.string_thing, 'hi!') + assert_equal(ret.struct_thing.i32_thing, 4) + assert_equal(ret.i32_thing, 10) + + assert_kind_of(Thrift::Test::Xtruct, ret.struct_thing) + assert_kind_of(Thrift::Test::Xtruct2, ret) + end + + def test_insane + insane = Thrift::Test::Insanity.new({ + 'userMap' => { Thrift::Test::Numberz::ONE => 44 }, + 'xtructs' => [get_struct, + Thrift::Test::Xtruct.new({ + 'string_thing' => 'hi again', + 'i32_thing' => 12 + }) + ] + }) + + ret = @client.testInsanity(insane) + + assert_not_nil(ret[44]) + assert_not_nil(ret[44][1]) + + struct = ret[44][1] + + assert_equal(struct.userMap[Thrift::Test::Numberz::ONE], 44) + assert_equal(struct.xtructs[1].string_thing, 'hi again') + assert_equal(struct.xtructs[1].i32_thing, 12) + + assert_kind_of(Hash, struct.userMap) + assert_kind_of(Array, struct.xtructs) + assert_kind_of(Thrift::Test::Insanity, struct) + end + + def test_map_map + ret = @client.testMapMap(4) + assert_kind_of(Hash, ret) + assert_equal(ret, { 4 => { 4 => 4}}) + end + + def test_exception + assert_raise Thrift::Test::Xception do + @client.testException('foo') + end + end +end + diff --git a/test/rb/integration/simple_server.rb b/test/rb/integration/simple_server.rb new file mode 100644 index 00000000..3518d2e1 --- /dev/null +++ b/test/rb/integration/simple_server.rb @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +$:.push File.dirname(__FILE__) + '/../gen-rb' +$:.push File.join(File.dirname(__FILE__), '../../../lib/rb/lib') + +require 'thrift' +require 'ThriftTest' + +class SimpleHandler + [:testString, :testByte, :testI32, :testI64, :testDouble, + :testStruct, :testMap, :testSet, :testList, :testNest, + :testEnum, :testTypedef].each do |meth| + + define_method(meth) do |thing| + thing + end + + end + + def testInsanity(thing) + num, uid = thing.userMap.find { true } + return {uid => {num => thing}} + end + + def testMapMap(thing) + return {thing => {thing => thing}} + end + + def testEnum(thing) + return thing + end + + def testTypedef(thing) + return thing + end + + def testException(thing) + raise Thrift::Test::Xception, :message => 'error' + end +end + +@handler = SimpleHandler.new +@processor = Thrift::Test::ThriftTest::Processor.new(@handler) +@transport = Thrift::ServerSocket.new(9090) +@server = Thrift::ThreadedServer.new(@processor, @transport) + +@server.serve diff --git a/test/rb/integration/test_simple_handler.rb b/test/rb/integration/test_simple_handler.rb new file mode 100644 index 00000000..c34aa7e5 --- /dev/null +++ b/test/rb/integration/test_simple_handler.rb @@ -0,0 +1,211 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +require File.join(File.dirname(__FILE__), '../test_helper') + +require 'thrift' +require 'ThriftTest' + +class TestHandler + [:testString, :testByte, :testI32, :testI64, :testDouble, + :testStruct, :testMap, :testSet, :testList, :testNest, + :testEnum, :testTypedef].each do |meth| + + define_method(meth) do |thing| + thing + end + + end + + def testInsanity(thing) + num, uid = thing.userMap.find { true } + return {uid => {num => thing}} + end + + def testMapMap(thing) + return {thing => {thing => thing}} + end + + def testEnum(thing) + return thing + end + + def testTypedef(thing) + return thing + end + + def testException(thing) + raise Thrift::Test::Xception, :message => 'error' + end + +end +class TestThrift < Test::Unit::TestCase + + @@INIT = nil + + def setup + if @@INIT.nil? + # Initialize the server + @handler = TestHandler.new() + @processor = Thrift::Test::ThriftTest::Processor.new(@handler) + @transport = Thrift::ServerSocket.new(9090) + @server = Thrift::ThreadedServer.new(@processor, @transport) + + @thread = Thread.new { @server.serve } + + # And the Client + @socket = Thrift::Socket.new('localhost', 9090) + @protocol = Thrift::BinaryProtocol.new(@socket) + @client = Thrift::Test::ThriftTest::Client.new(@protocol) + @socket.open + end + end + + def test_string + assert_equal(@client.testString('string'), 'string') + end + + def test_byte + val = 8 + assert_equal(@client.testByte(val), val) + assert_equal(@client.testByte(-val), -val) + end + + def test_i32 + val = 32 + assert_equal(@client.testI32(val), val) + assert_equal(@client.testI32(-val), -val) + end + + def test_i64 + val = 64 + assert_equal(@client.testI64(val), val) + assert_equal(@client.testI64(-val), -val) + end + + def test_double + val = 3.14 + assert_equal(@client.testDouble(val), val) + assert_equal(@client.testDouble(-val), -val) + assert_kind_of(Float, @client.testDouble(val)) + end + + def test_map + val = {1 => 1, 2 => 2, 3 => 3} + assert_equal(@client.testMap(val), val) + assert_kind_of(Hash, @client.testMap(val)) + end + + def test_list + val = [1,2,3,4,5] + assert_equal(@client.testList(val), val) + assert_kind_of(Array, @client.testList(val)) + end + + def test_enum + val = Thrift::Test::Numberz::SIX + ret = @client.testEnum(val) + + assert_equal(ret, 6) + assert_kind_of(Fixnum, ret) + end + + def test_typedef + #UserId testTypedef(1: UserId thing), + true + end + + def test_set + val = Set.new([1, 2, 3]) + assert_equal(val, @client.testSet(val)) + assert_kind_of(Set, @client.testSet(val)) + end + + def get_struct + Thrift::Test::Xtruct.new({'string_thing' => 'hi!', 'i32_thing' => 4 }) + end + + def test_struct + ret = @client.testStruct(get_struct) + + assert_nil(ret.byte_thing, nil) + assert_nil(ret.i64_thing, nil) + assert_equal(ret.string_thing, 'hi!') + assert_equal(ret.i32_thing, 4) + assert_kind_of(Thrift::Test::Xtruct, ret) + end + + def test_nest + struct2 = Thrift::Test::Xtruct2.new({'struct_thing' => get_struct, 'i32_thing' => 10}) + + ret = @client.testNest(struct2) + + assert_nil(ret.struct_thing.byte_thing, nil) + assert_nil(ret.struct_thing.i64_thing, nil) + assert_equal(ret.struct_thing.string_thing, 'hi!') + assert_equal(ret.struct_thing.i32_thing, 4) + assert_equal(ret.i32_thing, 10) + + assert_kind_of(Thrift::Test::Xtruct, ret.struct_thing) + assert_kind_of(Thrift::Test::Xtruct2, ret) + end + + def test_insane + insane = Thrift::Test::Insanity.new({ + 'userMap' => { Thrift::Test::Numberz::ONE => 44 }, + 'xtructs' => [get_struct, + Thrift::Test::Xtruct.new({ + 'string_thing' => 'hi again', + 'i32_thing' => 12 + }) + ] + }) + + ret = @client.testInsanity(insane) + + assert_not_nil(ret[44]) + assert_not_nil(ret[44][1]) + + struct = ret[44][1] + + assert_equal(struct.userMap[Thrift::Test::Numberz::ONE], 44) + assert_equal(struct.xtructs[1].string_thing, 'hi again') + assert_equal(struct.xtructs[1].i32_thing, 12) + + assert_kind_of(Hash, struct.userMap) + assert_kind_of(Array, struct.xtructs) + assert_kind_of(Thrift::Test::Insanity, struct) + end + + def test_map_map + ret = @client.testMapMap(4) + assert_kind_of(Hash, ret) + assert_equal(ret, { 4 => { 4 => 4}}) + end + + def test_exception + assert_raise Thrift::Test::Xception do + @client.testException('foo') + end + end + + def teardown + end + +end diff --git a/test/rb/test_helper.rb b/test/rb/test_helper.rb new file mode 100644 index 00000000..c1ed779e --- /dev/null +++ b/test/rb/test_helper.rb @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +$:.unshift File.dirname(__FILE__) + '/gen-rb' +$:.unshift File.join(File.dirname(__FILE__), '../../lib/rb/lib') +$:.unshift File.join(File.dirname(__FILE__), '../../lib/rb/ext') + +require 'test/unit' + +module Thrift + module Struct + def ==(other) + return false unless other.is_a? self.class + self.class.const_get(:FIELDS).collect {|fid, data| data[:name] }.all? do |field| + send(field) == other.send(field) + end + end + end +end diff --git a/test/rb/test_suite.rb b/test/rb/test_suite.rb new file mode 100644 index 00000000..b157c2c5 --- /dev/null +++ b/test/rb/test_suite.rb @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +Dir["{core,generation}/**/*.rb"].each {|f| require f } \ No newline at end of file diff --git a/test/threads/Makefile b/test/threads/Makefile new file mode 100644 index 00000000..14f1a589 --- /dev/null +++ b/test/threads/Makefile @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Default target is everything + +ifndef thrift_home +thrift_home=../../ +endif #thrift_home + +target: all + +ifndef boost_home +boost_home=/usr/local/include/boost-1_33_1 +endif #boost_home +target: all + +include_paths = $(thrift_home)/lib/cpp/src \ + $(boost_home) + +include_flags = $(patsubst %,-I%, $(include_paths)) + +# Tools +ifndef THRIFT +THRIFT = ../../compiler/cpp/thrift +endif # THRIFT + +CC = g++ +LD = g++ + +# Compiler flags +LFL = -L$(thrift_home)/lib/cpp/.libs -lthrift +CCFL = -Wall -O3 -g -I./gen-cpp $(include_flags) +CFL = $(CCFL) $(LFL) + +all: server client + +stubs: ThreadsTest.thrift + $(THRIFT) --gen cpp --gen py ThreadsTest.thrift + +server: stubs + g++ -o ThreadsServer $(CFL) ThreadsServer.cpp ./gen-cpp/ThreadsTest.cpp ./gen-cpp/ThreadsTest_types.cpp + +client: stubs + g++ -o ThreadsClient $(CFL) ThreadsClient.cpp ./gen-cpp/ThreadsTest.cpp ./gen-cpp/ThreadsTest_types.cpp + +clean: + $(RM) -r *.o ThreadsServer ThreadsClient gen-cpp gen-py diff --git a/test/threads/ThreadsClient.cpp b/test/threads/ThreadsClient.cpp new file mode 100644 index 00000000..85274a63 --- /dev/null +++ b/test/threads/ThreadsClient.cpp @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// This autogenerated skeleton file illustrates how to build a server. +// You should copy it to another filename to avoid overwriting it. + +#include "ThreadsTest.h" +#include +#include +#include +#include +#include +#include +#include + +using boost::shared_ptr; +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace apache::thrift::server; +using namespace apache::thrift::concurrency; + +int main(int argc, char **argv) { + int port = 9090; + std::string host = "localhost"; + + shared_ptr transport(new TSocket(host, port)); + shared_ptr protocol(new TBinaryProtocol(transport)); + + transport->open(); + + ThreadsTestClient client(protocol); + int val; + val = client.threadOne(5); + fprintf(stderr, "%d\n", val); + val = client.stop(); + fprintf(stderr, "%d\n", val); + val = client.threadTwo(5); + fprintf(stderr, "%d\n", val); + + transport->close(); + + fprintf(stderr, "done.\n"); + + return 0; +} + diff --git a/test/threads/ThreadsServer.cpp b/test/threads/ThreadsServer.cpp new file mode 100644 index 00000000..8734ee89 --- /dev/null +++ b/test/threads/ThreadsServer.cpp @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// This autogenerated skeleton file illustrates how to build a server. +// You should copy it to another filename to avoid overwriting it. + +#include "ThreadsTest.h" +#include +#include +#include +#include +#include +#include +#include +#include + +using boost::shared_ptr; +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace apache::thrift::server; +using namespace apache::thrift::concurrency; + + +class ThreadsTestHandler : virtual public ThreadsTestIf { + public: + ThreadsTestHandler() { + // Your initialization goes here + } + + int32_t threadOne(const int32_t sleep) { + // Your implementation goes here + printf("threadOne\n"); + go2sleep(1, sleep); + return 1; + } + + int32_t threadTwo(const int32_t sleep) { + // Your implementation goes here + printf("threadTwo\n"); + go2sleep(2, sleep); + return 1; + } + + int32_t threadThree(const int32_t sleep) { + // Your implementation goes here + printf("threadThree\n"); + go2sleep(3, sleep); + return 1; + } + + int32_t threadFour(const int32_t sleep) { + // Your implementation goes here + printf("threadFour\n"); + go2sleep(4, sleep); + return 1; + } + + int32_t stop() { + printf("stop\n"); + server_->stop(); + return 1; + } + + void setServer(boost::shared_ptr server) { + server_ = server; + } + +protected: + void go2sleep(int thread, int seconds) { + Monitor m; + for (int i = 0; i < seconds; ++i) { + fprintf(stderr, "Thread %d: sleep %d\n", thread, i); + try { + m.wait(1000); + } catch(TimedOutException& e) { + } + } + fprintf(stderr, "THREAD %d DONE\n", thread); + } + +private: + boost::shared_ptr server_; + +}; + +int main(int argc, char **argv) { + int port = 9090; + shared_ptr handler(new ThreadsTestHandler()); + shared_ptr processor(new ThreadsTestProcessor(handler)); + shared_ptr serverTransport(new TServerSocket(port)); + shared_ptr transportFactory(new TBufferedTransportFactory()); + shared_ptr protocolFactory(new TBinaryProtocolFactory()); + + /* + shared_ptr threadManager = + ThreadManager::newSimpleThreadManager(10); + shared_ptr threadFactory = + shared_ptr(new PosixThreadFactory()); + threadManager->threadFactory(threadFactory); + threadManager->start(); + + shared_ptr server = + shared_ptr(new TThreadPoolServer(processor, + serverTransport, + transportFactory, + protocolFactory, + threadManager)); + */ + + shared_ptr server = + shared_ptr(new TThreadedServer(processor, + serverTransport, + transportFactory, + protocolFactory)); + + handler->setServer(server); + + server->serve(); + + fprintf(stderr, "done.\n"); + + return 0; +} + diff --git a/test/threads/ThreadsTest.thrift b/test/threads/ThreadsTest.thrift new file mode 100644 index 00000000..caa93460 --- /dev/null +++ b/test/threads/ThreadsTest.thrift @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +service ThreadsTest { + i32 threadOne(1: i32 sleep=15), + i32 threadTwo(2: i32 sleep=15), + i32 threadThree(3: i32 sleep=15), + i32 threadFour(4: i32 sleep=15) + + i32 stop(); + +} diff --git a/tutorial/README b/tutorial/README new file mode 100644 index 00000000..a29f977b --- /dev/null +++ b/tutorial/README @@ -0,0 +1,42 @@ +Thrift Tutorial + +License +======= + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Tutorial +======== + +1) First things first, you'll need to install the Thrift compiler and the + language libraries. Do that using the instructions in the top level + README file. + +2) Read tutorial.thrift to learn about the syntax of a Thrift file + +3) Compile the code for the language of your choice: + + $ thrift + $ thrift -r --gen cpp tutorial.thrift + +4) Take a look at the generated code. + +5) Look in the language directories for sample client/server code. + +6) That's about it for now. This tutorial is intentionally brief. It should be + just enough to get you started and ready to build your own project. diff --git a/tutorial/cpp/CppClient.cpp b/tutorial/cpp/CppClient.cpp new file mode 100644 index 00000000..a3f17fee --- /dev/null +++ b/tutorial/cpp/CppClient.cpp @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include +#include +#include + +#include "../gen-cpp/Calculator.h" + +using namespace std; +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; + +using namespace tutorial; +using namespace shared; + +using namespace boost; + +int main(int argc, char** argv) { + shared_ptr socket(new TSocket("localhost", 9090)); + shared_ptr transport(new TBufferedTransport(socket)); + shared_ptr protocol(new TBinaryProtocol(transport)); + CalculatorClient client(protocol); + + try { + transport->open(); + + client.ping(); + printf("ping()\n"); + + int32_t sum = client.add(1,1); + printf("1+1=%d\n", sum); + + Work work; + work.op = DIVIDE; + work.num1 = 1; + work.num2 = 0; + + try { + int32_t quotient = client.calculate(1, work); + printf("Whoa? We can divide by zero!\n"); + } catch (InvalidOperation &io) { + printf("InvalidOperation: %s\n", io.why.c_str()); + } + + work.op = SUBTRACT; + work.num1 = 15; + work.num2 = 10; + int32_t diff = client.calculate(1, work); + printf("15-10=%d\n", diff); + + // Note that C++ uses return by reference for complex types to avoid + // costly copy construction + SharedStruct ss; + client.getStruct(ss, 1); + printf("Check log: %s\n", ss.value.c_str()); + + transport->close(); + } catch (TException &tx) { + printf("ERROR: %s\n", tx.what()); + } + +} diff --git a/tutorial/cpp/CppServer.cpp b/tutorial/cpp/CppServer.cpp new file mode 100644 index 00000000..23c2b833 --- /dev/null +++ b/tutorial/cpp/CppServer.cpp @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "../gen-cpp/Calculator.h" + +using namespace std; +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; +using namespace apache::thrift::server; + +using namespace tutorial; +using namespace shared; + +using namespace boost; + +class CalculatorHandler : public CalculatorIf { + public: + CalculatorHandler() {} + + void ping() { + printf("ping()\n"); + } + + int32_t add(const int32_t n1, const int32_t n2) { + printf("add(%d,%d)\n", n1, n2); + return n1 + n2; + } + + int32_t calculate(const int32_t logid, const Work &work) { + printf("calculate(%d,{%d,%d,%d})\n", logid, work.op, work.num1, work.num2); + int32_t val; + + switch (work.op) { + case ADD: + val = work.num1 + work.num2; + break; + case SUBTRACT: + val = work.num1 - work.num2; + break; + case MULTIPLY: + val = work.num1 * work.num2; + break; + case DIVIDE: + if (work.num2 == 0) { + InvalidOperation io; + io.what = work.op; + io.why = "Cannot divide by 0"; + throw io; + } + val = work.num1 / work.num2; + break; + default: + InvalidOperation io; + io.what = work.op; + io.why = "Invalid Operation"; + throw io; + } + + SharedStruct ss; + ss.key = logid; + char buffer[12]; + snprintf(buffer, sizeof(buffer), "%d", val); + ss.value = buffer; + + log[logid] = ss; + + return val; + } + + void getStruct(SharedStruct &ret, const int32_t logid) { + printf("getStruct(%d)\n", logid); + ret = log[logid]; + } + + void zip() { + printf("zip()\n"); + } + +protected: + map log; + +}; + +int main(int argc, char **argv) { + + shared_ptr protocolFactory(new TBinaryProtocolFactory()); + shared_ptr handler(new CalculatorHandler()); + shared_ptr processor(new CalculatorProcessor(handler)); + shared_ptr serverTransport(new TServerSocket(9090)); + shared_ptr transportFactory(new TBufferedTransportFactory()); + + TSimpleServer server(processor, + serverTransport, + transportFactory, + protocolFactory); + + + /** + * Or you could do one of these + + shared_ptr threadManager = + ThreadManager::newSimpleThreadManager(workerCount); + shared_ptr threadFactory = + shared_ptr(new PosixThreadFactory()); + threadManager->threadFactory(threadFactory); + threadManager->start(); + TThreadPoolServer server(processor, + serverTransport, + transportFactory, + protocolFactory, + threadManager); + + TThreadedServer server(processor, + serverTransport, + transportFactory, + protocolFactory); + + */ + + printf("Starting the server...\n"); + server.serve(); + printf("done.\n"); + return 0; +} diff --git a/tutorial/cpp/Makefile b/tutorial/cpp/Makefile new file mode 100644 index 00000000..e834dee2 --- /dev/null +++ b/tutorial/cpp/Makefile @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +BOOST_DIR = /usr/local/boost/include/boost-1_33_1/ +THRIFT_DIR = /usr/local/include/thrift +LIB_DIR = /usr/local/lib + +GEN_SRC = ../gen-cpp/SharedService.cpp ../gen-cpp/shared_types.cpp ../gen-cpp/tutorial_types.cpp ../gen-cpp/Calculator.cpp + +default: server client + +server: CppServer.cpp + g++ -o CppServer -I${THRIFT_DIR} -I${BOOST_DIR} -I../gen-cpp -L${LIB_DIR} -lthrift CppServer.cpp ${GEN_SRC} + +client: CppClient.cpp + g++ -o CppClient -I${THRIFT_DIR} -I${BOOST_DIR} -I../gen-cpp -L${LIB_DIR} -lthrift CppClient.cpp ${GEN_SRC} + +clean: + $(RM) -r CppClient CppServer diff --git a/tutorial/erl/client.erl b/tutorial/erl/client.erl new file mode 100644 index 00000000..97803349 --- /dev/null +++ b/tutorial/erl/client.erl @@ -0,0 +1,74 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(client). + +-include("calculator_thrift.hrl"). + +-export([t/0]). + +p(X) -> + io:format("~p~n", [X]), + ok. + +t() -> + Port = 9999, + + {ok, Client} = thrift_client:start_link("127.0.0.1", + Port, + calculator_thrift), + + thrift_client:call(Client, ping, []), + io:format("ping~n", []), + + {ok, Sum} = thrift_client:call(Client, add, [1, 1]), + io:format("1+1=~p~n", [Sum]), + + {ok, Sum1} = thrift_client:call(Client, add, [1, 4]), + io:format("1+4=~p~n", [Sum1]), + + Work = #work{op=?tutorial_SUBTRACT, + num1=15, + num2=10}, + {ok, Diff} = thrift_client:call(Client, calculate, [1, Work]), + io:format("15-10=~p~n", [Diff]), + + {ok, Log} = thrift_client:call(Client, getStruct, [1]), + io:format("Log: ~p~n", [Log]), + + try + Work1 = #work{op=?tutorial_DIVIDE, + num1=1, + num2=0}, + {ok, _Quot} = thrift_client:call(Client, calculate, [2, Work1]), + + io:format("LAME: exception handling is broken~n", []) + catch + Z -> + io:format("Got exception where expecting - the " ++ + "following is NOT a problem!!!~n"), + p(Z) + end, + + + {ok, ok} = thrift_client:call(Client, zip, []), + io:format("zip~n", []), + + ok = thrift_client:close(Client), + ok. diff --git a/tutorial/erl/client.sh b/tutorial/erl/client.sh new file mode 120000 index 00000000..a417e0da --- /dev/null +++ b/tutorial/erl/client.sh @@ -0,0 +1 @@ +server.sh \ No newline at end of file diff --git a/tutorial/erl/server.erl b/tutorial/erl/server.erl new file mode 100644 index 00000000..5a994ce7 --- /dev/null +++ b/tutorial/erl/server.erl @@ -0,0 +1,82 @@ +%% +%% Licensed to the Apache Software Foundation (ASF) under one +%% or more contributor license agreements. See the NOTICE file +%% distributed with this work for additional information +%% regarding copyright ownership. The ASF licenses this file +%% to you under the Apache License, Version 2.0 (the +%% "License"); you may not use this file except in compliance +%% with the License. You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(server). + +-include("calculator_thrift.hrl"). + +-export([start/0, start/1, handle_function/2, + stop/1, ping/0, add/2, calculate/2, getStruct/1, zip/0]). + +debug(Format, Data) -> + error_logger:info_msg(Format, Data). + +ping() -> + debug("ping()",[]), + ok. + +add(N1, N2) -> + debug("add(~p,~p)",[N1,N2]), + N1+N2. + +calculate(Logid, Work) -> + { Op, Num1, Num2 } = { Work#work.op, Work#work.num1, Work#work.num2 }, + debug("calculate(~p, {~p,~p,~p})", [Logid, Op, Num1, Num2]), + case Op of + ?tutorial_ADD -> Num1 + Num2; + ?tutorial_SUBTRACT -> Num1 - Num2; + ?tutorial_MULTIPLY -> Num1 * Num2; + + ?tutorial_DIVIDE when Num2 == 0 -> + throw(#invalidOperation{what=Op, why="Cannot divide by 0"}); + ?tutorial_DIVIDE -> + Num1 div Num2; + + _Else -> + throw(#invalidOperation{what=Op, why="Invalid operation"}) + end. + +getStruct(Key) -> + debug("getStruct(~p)", [Key]), + #sharedStruct{key=Key, value="RARG"}. + +zip() -> + debug("zip", []), + ok. + +%% + +start() -> + start(9999). + +start(Port) -> + Handler = ?MODULE, + thrift_socket_server:start([{handler, Handler}, + {service, calculator_thrift}, + {port, Port}, + {name, tutorial_server}]). + +stop(Server) -> + thrift_socket_server:stop(Server). + +handle_function(Function, Args) when is_atom(Function), is_tuple(Args) -> + case apply(?MODULE, Function, tuple_to_list(Args)) of + ok -> ok; + Reply -> {reply, Reply} + end. diff --git a/tutorial/erl/server.sh b/tutorial/erl/server.sh new file mode 100755 index 00000000..106c89e9 --- /dev/null +++ b/tutorial/erl/server.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +ERL_THRIFT=../../lib/erl + +if ! [ -d ${ERL_THRIFT}/ebin ]; then + echo "Please build the Thrift library by running \`make' in ${ERL_THRIFT}" + exit 1 +fi + +if ! [ -d ../gen-erl ]; then + echo "Please run thrift first to generate ../gen-erl/" + exit 1 +fi + + +erlc -I ${ERL_THRIFT}/include -I ../gen-erl -o ../gen-erl ../gen-erl/*.erl && + erlc -I ${ERL_THRIFT}/include -I ../gen-erl *.erl && + erl +K true -pa ${ERL_THRIFT}/ebin -pa ../gen-erl diff --git a/tutorial/java/JavaClient b/tutorial/java/JavaClient new file mode 100755 index 00000000..68d87b81 --- /dev/null +++ b/tutorial/java/JavaClient @@ -0,0 +1,22 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +java -cp tutorial.jar:/usr/local/lib/libthrift.jar JavaClient diff --git a/tutorial/java/JavaServer b/tutorial/java/JavaServer new file mode 100755 index 00000000..89616001 --- /dev/null +++ b/tutorial/java/JavaServer @@ -0,0 +1,22 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +java -cp tutorial.jar:/usr/local/lib/libthrift.jar JavaServer diff --git a/tutorial/java/build.xml b/tutorial/java/build.xml new file mode 100644 index 00000000..0ec1ea40 --- /dev/null +++ b/tutorial/java/build.xml @@ -0,0 +1,47 @@ + + + + Thrift Tutorial + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tutorial/java/src/JavaClient.java b/tutorial/java/src/JavaClient.java new file mode 100644 index 00000000..5dc70ed5 --- /dev/null +++ b/tutorial/java/src/JavaClient.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Generated code +import tutorial.*; +import shared.*; + +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; + +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.ArrayList; + +public class JavaClient { + public static void main(String [] args) { + try { + + TTransport transport = new TSocket("localhost", 9090); + TProtocol protocol = new TBinaryProtocol(transport); + Calculator.Client client = new Calculator.Client(protocol); + + transport.open(); + + client.ping(); + System.out.println("ping()"); + + int sum = client.add(1,1); + System.out.println("1+1=" + sum); + + Work work = new Work(); + + work.op = Operation.DIVIDE; + work.num1 = 1; + work.num2 = 0; + try { + int quotient = client.calculate(1, work); + System.out.println("Whoa we can divide by 0"); + } catch (InvalidOperation io) { + System.out.println("Invalid operation: " + io.why); + } + + work.op = Operation.SUBTRACT; + work.num1 = 15; + work.num2 = 10; + try { + int diff = client.calculate(1, work); + System.out.println("15-10=" + diff); + } catch (InvalidOperation io) { + System.out.println("Invalid operation: " + io.why); + } + + SharedStruct log = client.getStruct(1); + System.out.println("Check log: " + log.value); + + transport.close(); + + } catch (TException x) { + x.printStackTrace(); + } + + } + +} diff --git a/tutorial/java/src/JavaServer.java b/tutorial/java/src/JavaServer.java new file mode 100644 index 00000000..14440eb7 --- /dev/null +++ b/tutorial/java/src/JavaServer.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.server.TServer; +import org.apache.thrift.server.TSimpleServer; +import org.apache.thrift.transport.TServerSocket; +import org.apache.thrift.transport.TServerTransport; + +// Generated code +import tutorial.*; +import shared.*; + +import java.util.HashMap; + +public class JavaServer { + + public static class CalculatorHandler implements Calculator.Iface { + + private HashMap log; + + public CalculatorHandler() { + log = new HashMap(); + } + + public void ping() { + System.out.println("ping()"); + } + + public int add(int n1, int n2) { + System.out.println("add(" + n1 + "," + n2 + ")"); + return n1 + n2; + } + + public int calculate(int logid, Work work) throws InvalidOperation { + System.out.println("calculate(" + logid + ", {" + work.op + "," + work.num1 + "," + work.num2 + "})"); + int val = 0; + switch (work.op) { + case Operation.ADD: + val = work.num1 + work.num2; + break; + case Operation.SUBTRACT: + val = work.num1 - work.num2; + break; + case Operation.MULTIPLY: + val = work.num1 * work.num2; + break; + case Operation.DIVIDE: + if (work.num2 == 0) { + InvalidOperation io = new InvalidOperation(); + io.what = work.op; + io.why = "Cannot divide by 0"; + throw io; + } + val = work.num1 / work.num2; + break; + default: + InvalidOperation io = new InvalidOperation(); + io.what = work.op; + io.why = "Unknown operation"; + throw io; + } + + SharedStruct entry = new SharedStruct(); + entry.key = logid; + entry.value = Integer.toString(val); + log.put(logid, entry); + + return val; + } + + public SharedStruct getStruct(int key) { + System.out.println("getStruct(" + key + ")"); + return log.get(key); + } + + public void zip() { + System.out.println("zip()"); + } + + } + + public static void main(String [] args) { + try { + CalculatorHandler handler = new CalculatorHandler(); + Calculator.Processor processor = new Calculator.Processor(handler); + TServerTransport serverTransport = new TServerSocket(9090); + TServer server = new TSimpleServer(processor, serverTransport); + + // Use this for a multithreaded server + // server = new TThreadPoolServer(processor, serverTransport); + + System.out.println("Starting the server..."); + server.serve(); + + } catch (Exception x) { + x.printStackTrace(); + } + System.out.println("done."); + } +} diff --git a/tutorial/perl/PerlClient.pl b/tutorial/perl/PerlClient.pl new file mode 100644 index 00000000..1d596568 --- /dev/null +++ b/tutorial/perl/PerlClient.pl @@ -0,0 +1,82 @@ +#!/usr/bin/env perl + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +use strict; +use warnings; + +use lib '../../lib/perl/lib'; +use lib '../gen-perl'; + +use Thrift; +use Thrift::BinaryProtocol; +use Thrift::Socket; +use Thrift::BufferedTransport; + +use shared::SharedService; +use tutorial::Calculator; +use shared::Types; +use tutorial::Types; + +use Data::Dumper; + +my $socket = new Thrift::Socket('localhost',9090); +my $transport = new Thrift::BufferedTransport($socket,1024,1024); +my $protocol = new Thrift::BinaryProtocol($transport); +my $client = new tutorial::CalculatorClient($protocol); + + +eval{ + $transport->open(); + + $client->ping(); + print "ping()\n"; + + + my $sum = $client->add(1,1); + print "1+1=$sum\n"; + + my $work = new tutorial::Work(); + + $work->op(tutorial::Operation::DIVIDE); + $work->num1(1); + $work->num2(0); + + eval { + $client->calculate(1, $work); + print "Whoa! We can divide by zero?\n"; + }; if($@) { + warn "InvalidOperation: ".Dumper($@); + } + + $work->op(tutorial::Operation::SUBTRACT); + $work->num1(15); + $work->num2(10); + my $diff = $client->calculate(1, $work); + print "15-10=$diff\n"; + + my $log = $client->getStruct(1); + print "Log: $log->{value}\n"; + + $transport->close(); + +}; if($@){ + warn(Dumper($@)); +} diff --git a/tutorial/php/PhpClient.php b/tutorial/php/PhpClient.php new file mode 100755 index 00000000..c5c08101 --- /dev/null +++ b/tutorial/php/PhpClient.php @@ -0,0 +1,92 @@ +#!/usr/bin/env php +open(); + + $client->ping(); + print "ping()\n"; + + $sum = $client->add(1,1); + print "1+1=$sum\n"; + + $work = new tutorial_Work(); + + $work->op = tutorial_Operation::DIVIDE; + $work->num1 = 1; + $work->num2 = 0; + + try { + $client->calculate(1, $work); + print "Whoa! We can divide by zero?\n"; + } catch (tutorial_InvalidOperation $io) { + print "InvalidOperation: $io->why\n"; + } + + $work->op = tutorial_Operation::SUBTRACT; + $work->num1 = 15; + $work->num2 = 10; + $diff = $client->calculate(1, $work); + print "15-10=$diff\n"; + + $log = $client->getStruct(1); + print "Log: $log->value\n"; + + $transport->close(); + +} catch (TException $tx) { + print 'TException: '.$tx->getMessage()."\n"; +} + +?> diff --git a/tutorial/php/PhpServer.php b/tutorial/php/PhpServer.php new file mode 100755 index 00000000..9482c649 --- /dev/null +++ b/tutorial/php/PhpServer.php @@ -0,0 +1,132 @@ +#!/usr/bin/env php +op}, {$w->num1}, {$w->num2}})"); + switch ($w->op) { + case tutorial_Operation::ADD: + $val = $w->num1 + $w->num2; + break; + case tutorial_Operation::SUBTRACT: + $val = $w->num1 - $w->num2; + break; + case tutorial_Operation::MULTIPLY: + $val = $w->num1 * $w->num2; + break; + case tutorial_Operation::DIVIDE: + if ($w->num2 == 0) { + $io = new tutorial_InvalidOperation(); + $io->what = $w->op; + $io->why = "Cannot divide by 0"; + throw $io; + } + $val = $w->num1 / $w->num2; + break; + default: + $io = new tutorial_InvalidOperation(); + $io->what = $w->op; + $io->why = "Invalid Operation"; + throw $io; + } + + $log = new SharedStruct(); + $log->key = $logid; + $log->value = (string)$val; + $this->log[$logid] = $log; + + return $val; + } + + public function getStruct($key) { + error_log("getStruct({$key})"); + // This actually doesn't work because the PHP interpreter is + // restarted for every request. + //return $this->log[$key]; + return new SharedStruct(array("key" => $key, "value" => "PHP is stateless!")); + } + + public function zip() { + error_log("zip()"); + } + +}; + +header('Content-Type', 'application/x-thrift'); +if (php_sapi_name() == 'cli') { + echo "\r\n"; +} + +$handler = new CalculatorHandler(); +$processor = new CalculatorProcessor($handler); + +$transport = new TBufferedTransport(new TPhpStream(TPhpStream::MODE_R | TPhpStream::MODE_W)); +$protocol = new TBinaryProtocol($transport, true, true); + +$transport->open(); +$processor->process($protocol, $protocol); +$transport->close(); diff --git a/tutorial/php/runserver.py b/tutorial/php/runserver.py new file mode 100755 index 00000000..ae29fed9 --- /dev/null +++ b/tutorial/php/runserver.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import os +import BaseHTTPServer +import CGIHTTPServer + +# chdir(2) into the tutorial directory. +os.chdir(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + +class Handler(CGIHTTPServer.CGIHTTPRequestHandler): + cgi_directories = ['/php'] + +BaseHTTPServer.HTTPServer(('', 8080), Handler).serve_forever() diff --git a/tutorial/py/PythonClient.py b/tutorial/py/PythonClient.py new file mode 100755 index 00000000..916e9157 --- /dev/null +++ b/tutorial/py/PythonClient.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys +sys.path.append('../gen-py') + +from tutorial import Calculator +from tutorial.ttypes import * + +from thrift import Thrift +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol + +try: + + # Make socket + transport = TSocket.TSocket('localhost', 9090) + + # Buffering is critical. Raw sockets are very slow + transport = TTransport.TBufferedTransport(transport) + + # Wrap in a protocol + protocol = TBinaryProtocol.TBinaryProtocol(transport) + + # Create a client to use the protocol encoder + client = Calculator.Client(protocol) + + # Connect! + transport.open() + + client.ping() + print 'ping()' + + sum = client.add(1,1) + print '1+1=%d' % (sum) + + work = Work() + + work.op = Operation.DIVIDE + work.num1 = 1 + work.num2 = 0 + + try: + quotient = client.calculate(1, work) + print 'Whoa? You know how to divide by zero?' + except InvalidOperation, io: + print 'InvalidOperation: %r' % io + + work.op = Operation.SUBTRACT + work.num1 = 15 + work.num2 = 10 + + diff = client.calculate(1, work) + print '15-10=%d' % (diff) + + log = client.getStruct(1) + print 'Check log: %s' % (log.value) + + # Close! + transport.close() + +except Thrift.TException, tx: + print '%s' % (tx.message) diff --git a/tutorial/py/PythonServer.py b/tutorial/py/PythonServer.py new file mode 100755 index 00000000..63f993bc --- /dev/null +++ b/tutorial/py/PythonServer.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import sys +sys.path.append('../gen-py') + +from tutorial import Calculator +from tutorial.ttypes import * + +from shared.ttypes import SharedStruct + +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol +from thrift.server import TServer + +class CalculatorHandler: + def __init__(self): + self.log = {} + + def ping(self): + print 'ping()' + + def add(self, n1, n2): + print 'add(%d,%d)' % (n1, n2) + return n1+n2 + + def calculate(self, logid, work): + print 'calculate(%d, %r)' % (logid, work) + + if work.op == Operation.ADD: + val = work.num1 + work.num2 + elif work.op == Operation.SUBTRACT: + val = work.num1 - work.num2 + elif work.op == Operation.MULTIPLY: + val = work.num1 * work.num2 + elif work.op == Operation.DIVIDE: + if work.num2 == 0: + x = InvalidOperation() + x.what = work.op + x.why = 'Cannot divide by 0' + raise x + val = work.num1 / work.num2 + else: + x = InvalidOperation() + x.what = work.op + x.why = 'Invalid operation' + raise x + + log = SharedStruct() + log.key = logid + log.value = '%d' % (val) + self.log[logid] = log + + return val + + def getStruct(self, key): + print 'getStruct(%d)' % (key) + return self.log[key] + + def zip(self): + print 'zip()' + +handler = CalculatorHandler() +processor = Calculator.Processor(handler) +transport = TSocket.TServerSocket(9090) +tfactory = TTransport.TBufferedTransportFactory() +pfactory = TBinaryProtocol.TBinaryProtocolFactory() + +server = TServer.TSimpleServer(processor, transport, tfactory, pfactory) + +# You could do one of these for a multithreaded server +#server = TServer.TThreadedServer(processor, transport, tfactory, pfactory) +#server = TServer.TThreadPoolServer(processor, transport, tfactory, pfactory) + +print 'Starting the server...' +server.serve() +print 'done.' diff --git a/tutorial/rb/RubyClient.rb b/tutorial/rb/RubyClient.rb new file mode 100755 index 00000000..8971fed9 --- /dev/null +++ b/tutorial/rb/RubyClient.rb @@ -0,0 +1,75 @@ +#!/usr/bin/env ruby + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +$:.push('../gen-rb') +$:.unshift '../../lib/rb/lib' + +require 'thrift' + +require 'Calculator' + +begin + port = ARGV[0] || 9090 + + transport = Thrift::BufferedTransport.new(Thrift::Socket.new('localhost', port)) + protocol = Thrift::BinaryProtocol.new(transport) + client = Calculator::Client.new(protocol) + + transport.open() + + client.ping() + print "ping()\n" + + sum = client.add(1,1) + print "1+1=", sum, "\n" + + sum = client.add(1,4) + print "1+4=", sum, "\n" + + work = Work.new() + + work.op = Operation::SUBTRACT + work.num1 = 15 + work.num2 = 10 + diff = client.calculate(1, work) + print "15-10=", diff, "\n" + + log = client.getStruct(1) + print "Log: ", log.value, "\n" + + begin + work.op = Operation::DIVIDE + work.num1 = 1 + work.num2 = 0 + quot = client.calculate(1, work) + puts "Whoa, we can divide by 0 now?" + rescue InvalidOperation => io + print "InvalidOperation: ", io.why, "\n" + end + + client.zip() + print "zip\n" + + transport.close() + +rescue Thrift::Exception => tx + print 'Thrift::Exception: ', tx.message, "\n" +end diff --git a/tutorial/rb/RubyServer.rb b/tutorial/rb/RubyServer.rb new file mode 100755 index 00000000..89eb3738 --- /dev/null +++ b/tutorial/rb/RubyServer.rb @@ -0,0 +1,95 @@ +#!/usr/bin/env ruby + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +$:.push('../gen-rb') +$:.unshift '../../lib/rb/lib' + +require 'thrift' + +require 'Calculator' +require 'shared_types' + +class CalculatorHandler + def initialize() + @log = {} + end + + def ping() + puts "ping()" + end + + def add(n1, n2) + print "add(", n1, ",", n2, ")\n" + return n1 + n2 + end + + def calculate(logid, work) + print "calculate(", logid, ", {", work.op, ",", work.num1, ",", work.num2,"})\n" + if work.op == Operation::ADD + val = work.num1 + work.num2 + elsif work.op == Operation::SUBTRACT + val = work.num1 - work.num2 + elsif work.op == Operation::MULTIPLY + val = work.num1 * work.num2 + elsif work.op == Operation::DIVIDE + if work.num2 == 0 + x = InvalidOperation.new() + x.what = work.op + x.why = "Cannot divide by 0" + raise x + end + val = work.num1 / work.num2 + else + x = InvalidOperation.new() + x.what = work.op + x.why = "Invalid operation" + raise x + end + + entry = SharedStruct.new() + entry.key = logid + entry.value = "#{val}" + @log[logid] = entry + + return val + + end + + def getStruct(key) + print "getStruct(", key, ")\n" + return @log[key] + end + + def zip() + print "zip\n" + end + +end + +handler = CalculatorHandler.new() +processor = Calculator::Processor.new(handler) +transport = Thrift::ServerSocket.new(9090) +transportFactory = Thrift::BufferedTransportFactory.new() +server = Thrift::SimpleServer.new(processor, transport, transportFactory) + +puts "Starting the server..." +server.serve() +puts "done." diff --git a/tutorial/shared.thrift b/tutorial/shared.thrift new file mode 100644 index 00000000..475e7f80 --- /dev/null +++ b/tutorial/shared.thrift @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * This Thrift file can be included by other Thrift files that want to share + * these definitions. + */ + +namespace cpp shared +namespace java shared +namespace perl shared + +struct SharedStruct { + 1: i32 key + 2: string value +} + +service SharedService { + SharedStruct getStruct(1: i32 key) +} diff --git a/tutorial/tutorial.thrift b/tutorial/tutorial.thrift new file mode 100644 index 00000000..86e433dd --- /dev/null +++ b/tutorial/tutorial.thrift @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +# Thrift Tutorial +# Mark Slee (mcslee@facebook.com) +# +# This file aims to teach you how to use Thrift, in a .thrift file. Neato. The +# first thing to notice is that .thrift files support standard shell comments. +# This lets you make your thrift file executable and include your Thrift build +# step on the top line. And you can place comments like this anywhere you like. +# +# Before running this file, you will need to have installed the thrift compiler +# into /usr/local/bin. + +/** + * The first thing to know about are types. The available types in Thrift are: + * + * bool Boolean, one byte + * byte Signed byte + * i16 Signed 16-bit integer + * i32 Signed 32-bit integer + * i64 Signed 64-bit integer + * double 64-bit floating point value + * string String + * binary Blob (byte array) + * map Map from one type to another + * list Ordered list of one type + * set Set of unique elements of one type + * + * Did you also notice that Thrift supports C style comments? + */ + +// Just in case you were wondering... yes. We support simple C comments too. + +/** + * Thrift files can reference other Thrift files to include common struct + * and service definitions. These are found using the current path, or by + * searching relative to any paths specified with the -I compiler flag. + * + * Included objects are accessed using the name of the .thrift file as a + * prefix. i.e. shared.SharedObject + */ +include "shared.thrift" + +/** + * Thrift files can namespace, package, or prefix their output in various + * target languages. + */ +namespace cpp tutorial +namespace java tutorial +namespace php tutorial +namespace perl tutorial +namespace smalltalk.category Thrift.Tutorial + +/** + * Thrift lets you do typedefs to get pretty names for your types. Standard + * C style here. + */ +typedef i32 MyInteger + +/** + * Thrift also lets you define constants for use across languages. Complex + * types and structs are specified using JSON notation. + */ +const i32 INT32CONSTANT = 9853 +const map MAPCONSTANT = {'hello':'world', 'goodnight':'moon'} + +/** + * You can define enums, which are just 32 bit integers. Values are optional + * and start at 1 if not supplied, C style again. + */ +enum Operation { + ADD = 1, + SUBTRACT = 2, + MULTIPLY = 3, + DIVIDE = 4 +} + +/** + * Structs are the basic complex data structures. They are comprised of fields + * which each have an integer identifier, a type, a symbolic name, and an + * optional default value. + * + * Fields can be declared "optional", which ensures they will not be included + * in the serialized output if they aren't set. Note that this requires some + * manual management in some languages. + */ +struct Work { + 1: i32 num1 = 0, + 2: i32 num2, + 3: Operation op, + 4: optional string comment, +} + +/** + * Structs can also be exceptions, if they are nasty. + */ +exception InvalidOperation { + 1: i32 what, + 2: string why +} + +/** + * Ahh, now onto the cool part, defining a service. Services just need a name + * and can optionally inherit from another service using the extends keyword. + */ +service Calculator extends shared.SharedService { + + /** + * A method definition looks like C code. It has a return type, arguments, + * and optionally a list of exceptions that it may throw. Note that argument + * lists and exception lists are specified using the exact same syntax as + * field lists in struct or exception definitions. + */ + + void ping(), + + i32 add(1:i32 num1, 2:i32 num2), + + i32 calculate(1:i32 logid, 2:Work w) throws (1:InvalidOperation ouch), + + /** + * This method has a oneway modifier. That means the client only makes + * a request and does not listen for any response at all. Oneway methods + * must be void. + */ + oneway void zip() + +} + +/** + * That just about covers the basics. Take a look in the test/ folder for more + * detailed examples. After you run this file, your generated code shows up + * in folders with names gen-. The generated code isn't too scary + * to look at. It even has pretty indentation. + */ -- 2.17.1