From ae756f4561047df087f7b1130d7e2b056b05e4b0 Mon Sep 17 00:00:00 2001 From: David Reiss Date: Tue, 10 Jun 2008 22:57:11 +0000 Subject: [PATCH] Read and write of structs, lists, maps, and sets git-svn-id: https://svn.apache.org/repos/asf/incubator/thrift/trunk@666375 13f79535-47bb-0310-9956-ffa450edef68 --- lib/alterl/include/thrift_protocol.hrl | 1 + lib/alterl/src/Makefile | 113 ++++++++++++++ lib/alterl/src/thrift.app.src | 46 ++++++ lib/alterl/src/thrift.appup.src | 1 + lib/alterl/src/thrift_binary_protocol.erl | 7 +- lib/alterl/src/thrift_processor.erl | 74 +++++---- lib/alterl/src/thrift_protocol.erl | 181 +++++++++++++++++++++- 7 files changed, 382 insertions(+), 41 deletions(-) create mode 100644 lib/alterl/src/Makefile create mode 100644 lib/alterl/src/thrift.app.src create mode 100644 lib/alterl/src/thrift.appup.src diff --git a/lib/alterl/include/thrift_protocol.hrl b/lib/alterl/include/thrift_protocol.hrl index 66222d62..3493f2ec 100644 --- a/lib/alterl/include/thrift_protocol.hrl +++ b/lib/alterl/include/thrift_protocol.hrl @@ -2,6 +2,7 @@ -define(THRIFT_PROTOCOL_INCLUDED, yea). -record(protocol_message_begin, {name, type, seqid}). +-record(protocol_struct_begin, {name}). -record(protocol_field_begin, {name, type, id}). -record(protocol_map_begin, {ktype, vtype, size}). -record(protocol_list_begin, {etype, size}). diff --git a/lib/alterl/src/Makefile b/lib/alterl/src/Makefile new file mode 100644 index 00000000..b42e0056 --- /dev/null +++ b/lib/alterl/src/Makefile @@ -0,0 +1,113 @@ +# $Id: Makefile,v 1.3 2004/08/13 16:35:59 mlogan Exp $ +# +include ../build/otp.mk +include ../build/colors.mk +include ../build/buildtargets.mk + +# ---------------------------------------------------- +# Application version +# ---------------------------------------------------- + +include ../vsn.mk +APP_NAME=thrift +PFX=thrift +VSN=$(THRIFT_VSN) + +# ---------------------------------------------------- +# Install directory specification +# WARNING: INSTALL_DIR the command to install a directory. +# INSTALL_DST is the target directory +# ---------------------------------------------------- +INSTALL_DST = $(ERLANG_OTP)/lib/$(APP_NAME)-$(VSN) + +# ---------------------------------------------------- +# Target Specs +# ---------------------------------------------------- + + +MODULES = $(shell find -name \*.erl | sed s:^\\./:: | sed s/\\.erl//) +MODULES_STRING_LIST = $(shell find -name \*.erl | sed s:^\\./:\": | sed s/\\.erl/\",/) + +HRL_FILES= +INTERNAL_HRL_FILES= $(APP_NAME).hrl +ERL_FILES= $(MODULES:%=%.erl) +DOC_FILES=$(ERL_FILES) + +APP_FILE= $(APP_NAME).app +APPUP_FILE= $(APP_NAME).appup + +APP_SRC= $(APP_FILE).src +APPUP_SRC= $(APPUP_FILE).src + +APP_TARGET= $(EBIN)/$(APP_FILE) +APPUP_TARGET= $(EBIN)/$(APPUP_FILE) + +BEAMS= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) +TARGET_FILES= $(BEAMS) $(APP_TARGET) $(APPUP_TARGET) + +WEB_TARGET=/var/yaws/www/$(APP_NAME) + +# ---------------------------------------------------- +# FLAGS +# ---------------------------------------------------- + +ERL_FLAGS += +ERL_COMPILE_FLAGS += -I../include -I../../fslib/include -I../../system_status/include + +# ---------------------------------------------------- +# Targets +# ---------------------------------------------------- + +all debug opt: $(EBIN) $(TARGET_FILES) + +#$(EBIN)/rm_logger.beam: $(APP_NAME).hrl +include ../build/docs.mk + +# Note: In the open-source build clean must not destroy the preloaded +# beam files. +clean: + rm -f $(TARGET_FILES) + rm -f *~ + rm -f core + rm -rf $(EBIN) + rm -rf *html + +$(EBIN): + mkdir $(EBIN) + +# ---------------------------------------------------- +# Special Build Targets +# ---------------------------------------------------- + +$(APP_TARGET): $(APP_SRC) ../vsn.mk $(BEAMS) + sed -e 's;%VSN%;$(VSN);' \ + -e 's;%PFX%;$(PFX);' \ + -e 's;%APP_NAME%;$(APP_NAME);' \ + -e 's;%MODULES%;%MODULES%$(MODULES_STRING_LIST);' \ + $< > $<".tmp" + sed -e 's/%MODULES%\(.*\),/\1/' \ + $<".tmp" > $@ + rm $<".tmp" + + +$(APPUP_TARGET): $(APPUP_SRC) ../vsn.mk + sed -e 's;%VSN%;$(VSN);' $< > $@ + +$(WEB_TARGET): ../markup/* + rm -rf $(WEB_TARGET) + mkdir $(WEB_TARGET) + cp -r ../markup/ $(WEB_TARGET) + cp -r ../skins/ $(WEB_TARGET) + +# ---------------------------------------------------- +# Install Target +# ---------------------------------------------------- + +install: all $(WEB_TARGET) +# $(INSTALL_DIR) $(INSTALL_DST)/src +# $(INSTALL_DATA) $(ERL_FILES) $(INSTALL_DST)/src +# $(INSTALL_DATA) $(INTERNAL_HRL_FILES) $(INSTALL_DST)/src +# $(INSTALL_DIR) $(INSTALL_DST)/include +# $(INSTALL_DATA) $(HRL_FILES) $(INSTALL_DST)/include +# $(INSTALL_DIR) $(INSTALL_DST)/ebin +# $(INSTALL_DATA) $(TARGET_FILES) $(INSTALL_DST)/ebin diff --git a/lib/alterl/src/thrift.app.src b/lib/alterl/src/thrift.app.src new file mode 100644 index 00000000..79055ca6 --- /dev/null +++ b/lib/alterl/src/thrift.app.src @@ -0,0 +1,46 @@ +%%% -*- mode:erlang -*- +{application, %APP_NAME%, + [ + % A quick description of the application. + {description, "Thrift bindings"}, + + % The version of the applicaton + {vsn, "%VSN%"}, + + % All modules used by the application. + {modules, [ + %MODULES% + ]}, + + % All of the registered names the application uses. This can be ignored. + {registered, []}, + + % Applications that are to be started prior to this one. This can be ignored + % leave it alone unless you understand it well and let the .rel files in + % your release handle this. + {applications, + [ + kernel, + stdlib + ]}, + + % OTP application loader will load, but not start, included apps. Again + % this can be ignored as well. To load but not start an application it + % is easier to include it in the .rel file followed by the atom 'none' + {included_applications, []}, + + % configuration parameters similar to those in the config file specified + % on the command line. can be fetched with gas:get_env + {env, [ + {term_width, 110}, + {force_one_line, false}, + {omit_fmt, ["thrift ~p:new(~s) = ~s"]}, + {gen_server_messages, true}, + {show_pid, true}, + {lookup, false} % DNS + ]}, + + % The Module and Args used to start this application. + {mod, {thrift_app, []}} + ] +}. diff --git a/lib/alterl/src/thrift.appup.src b/lib/alterl/src/thrift.appup.src new file mode 100644 index 00000000..54a63833 --- /dev/null +++ b/lib/alterl/src/thrift.appup.src @@ -0,0 +1 @@ +{"%VSN%",[],[]}. diff --git a/lib/alterl/src/thrift_binary_protocol.erl b/lib/alterl/src/thrift_binary_protocol.erl index 1593f0b8..6df9fdbf 100644 --- a/lib/alterl/src/thrift_binary_protocol.erl +++ b/lib/alterl/src/thrift_binary_protocol.erl @@ -85,7 +85,7 @@ write(This, #protocol_set_begin{ write(This, set_end) -> ok; -write(This, struct_begin) -> ok; +write(This, #protocol_struct_begin{}) -> ok; write(This, struct_end) -> ok; @@ -126,7 +126,8 @@ read(This, message_begin) -> #protocol_message_begin{name = Name, type = Type, seqid = SeqId}; - Err = {error, closed} -> Err + Err = {error, closed} -> Err; + Err = {error, ebadf} -> Err end; read(This, message_end) -> ok; @@ -140,7 +141,7 @@ read(This, field_begin) -> #protocol_field_begin{type = Type, id = 0}; % TODO(todd) 0 or undefined? {ok, Type} -> - Id = read(This, i16), + {ok, Id} = read(This, i16), #protocol_field_begin{type = Type, id = Id} end; diff --git a/lib/alterl/src/thrift_processor.erl b/lib/alterl/src/thrift_processor.erl index ea6ea2d3..fd6972db 100644 --- a/lib/alterl/src/thrift_processor.erl +++ b/lib/alterl/src/thrift_processor.erl @@ -26,39 +26,45 @@ init(IProt, OProt, Service, Handler) -> loop(State = #state{in_protocol = IProto, out_protocol = OProto}) -> - MessageBegin = thrift_protocol:read(IProto, message_begin), - io:format("Got message begin: ~p~n", [MessageBegin]), + case thrift_protocol:read(IProto, message_begin) of + #protocol_message_begin{name = Function, + type = ?tMessageType_CALL} -> + ok = handle_function(State, list_to_atom(binary_to_list(Function))), + loop(State); + {error, closed} -> + error_logger:info_msg("Processor finished~n"), + ok + end. + +handle_function(State = #state{in_protocol = IProto, + out_protocol = OProto}, + add) -> + io:format("Reading struct~n"), + {ok, Struct} = thrift_protocol:read(IProto, + {struct, [{1, i32}, + {2, i32}]}), + io:format("Struct: ~p~n", [Struct]), + + {A, B} = Struct, + + thrift_protocol:write(OProto, #protocol_message_begin{ + name = "addResult", + type = ?tMessageType_REPLY, + seqid = 0}), + thrift_protocol:write(OProto, {{struct, [{0, i32}]}, + {A + B}}), + thrift_protocol:write(OProto, message_end); + +handle_function(State = #state{in_protocol = IProto, + out_protocol = OProto}, + complexTest) -> + io:format("Reading struct~n"), + Struct = thrift_protocol:read( + IProto, + {struct, [{1, {struct, + [{1, {list, i32}}, + {2, {map, string, {struct, [{1, i16}]}}}]}}]}), + + io:format("Struct: ~p~n", [Struct]). - [ok, ok, ok, ok] = [thrift_protocol:read(IProto, X) - || X <- [struct_begin, field_stop, struct_end, message_end]], - io:format("Read everything okay!"), - Packets = - [ - #protocol_message_begin{name = "getServiceStatus", - type = ?tMessageType_REPLY, - seqid = 0}, - struct_begin, - #protocol_field_begin{name = "success", - type = ?tType_MAP, - id = 0}, - #protocol_map_begin{ktype = ?tType_STRING, - vtype = ?tType_STRING, - size = 2}, - {string, "Hello"}, - {string, "World"}, - {string, "foo"}, - {string, "bar"}, - field_stop, - map_end, - field_end, - field_stop, - struct_end, - message_end - ], - - Results = [thrift_protocol:write(OProto, Packet) || Packet <- Packets], - receive - _ -> - loop(State) - end. diff --git a/lib/alterl/src/thrift_protocol.erl b/lib/alterl/src/thrift_protocol.erl index 66e19b38..75c2d4bb 100644 --- a/lib/alterl/src/thrift_protocol.erl +++ b/lib/alterl/src/thrift_protocol.erl @@ -27,10 +27,6 @@ new(Module, Data) when is_atom(Module) -> data = Data}}. -write(#protocol{module = Module, - data = ModuleData}, Data) -> - Module:write(ModuleData, Data). - typeid_to_atom(?tType_STOP) -> field_stop; typeid_to_atom(?tType_VOID) -> void; typeid_to_atom(?tType_BOOL) -> bool; @@ -45,10 +41,92 @@ typeid_to_atom(?tType_MAP) -> map; typeid_to_atom(?tType_SET) -> set; typeid_to_atom(?tType_LIST) -> list. + +term_to_typeid(void) -> ?tType_VOID; +term_to_typeid(bool) -> ?tType_BOOL; +term_to_typeid(byte) -> ?tType_BYTE; +term_to_typeid(double) -> ?tType_DOUBLE; +term_to_typeid(i16) -> ?tType_I16; +term_to_typeid(i32) -> ?tType_I32; +term_to_typeid(i64) -> ?tType_I64; +term_to_typeid(string) -> ?tType_STRING; +term_to_typeid({struct, _}) -> ?tType_STRUCT; +term_to_typeid({map, _, _}) -> ?tType_MAP; +term_to_typeid({set, _}) -> ?tType_SET; +term_to_typeid({list, _}) -> ?tType_LIST. + + +%% Structure is like: +%% [{Fid, Type}, ...] +read(IProto, {struct, Structure}) when is_list(Structure) -> + SWithIndices = [{Fid, {Type, Index}} || + {{Fid, Type}, Index} <- + lists:zip(Structure, lists:seq(1, length(Structure)))], + % Fid -> {Type, Index} + SDict = dict:from_list(SWithIndices), + + + ok = read(IProto, struct_begin), + RDict = read_struct_loop(IProto, SDict, dict:new()), + + List = [case dict:find(Index, RDict) of + {ok, Val} -> Val; + error -> undefined + end || Index <- lists:seq(1, length(Structure))], + {ok, list_to_tuple(List)}; + +read(IProto, {list, Type}) -> + #protocol_list_begin{etype = EType, size = Size} = + read(IProto, list_begin), + List = [Result || {ok, Result} <- + [read(IProto, Type) || _X <- lists:seq(1, Size)]], + ok = read(IProto, list_end), + {ok, List}; + +read(IProto, {map, KeyType, ValType}) -> + #protocol_map_begin{size = Size} = + read(IProto, map_begin), + + List = [{Key, Val} || {{ok, Key}, {ok, Val}} <- + [{read(IProto, KeyType), + read(IProto, ValType)} || _X <- lists:seq(1, Size)]], + ok = read(IProto, map_end), + {ok, dict:from_list(List)}; + +read(IProto, {set, Type}) -> + #protocol_set_begin{etype = _EType, + size = Size} = + read(IProto, set_begin), + List = [Result || {ok, Result} <- + [read(IProto, Type) || _X <- lists:seq(1, Size)]], + ok = read(IProto, set_end), + {ok, sets:from_list(List)}; + read(#protocol{module = Module, data = ModuleData}, ProtocolType) -> Module:read(ModuleData, ProtocolType). +read_struct_loop(IProto, SDict, RDict) -> + #protocol_field_begin{type = FType, id = Fid, name = Name} = + thrift_protocol:read(IProto, field_begin), + case {FType, Fid} of + {?tType_STOP, _} -> + RDict; + _Else -> + case dict:find(Fid, SDict) of + {ok, {Type, Index}} -> + {ok, Val} = read(IProto, Type), + thrift_protocol:read(IProto, field_end), + NewRDict = dict:store(Index, Val, RDict), + read_struct_loop(IProto, SDict, NewRDict); + _Else2 -> + error_logger:info_msg("Skipping fid ~p~n", [Fid]), + FTypeAtom = thrift_protocol:typeid_to_atom(FType), + thrift_protocol:skip(IProto, FTypeAtom), + read(IProto, field_end), + read_struct_loop(IProto, SDict, RDict) + end + end. skip(Proto, struct) -> @@ -118,3 +196,98 @@ skip_list_loop(Proto, Map = #protocol_list_begin{etype = Etype, Map#protocol_list_begin{size = Size - 1}); 0 -> ok end. + + +%%-------------------------------------------------------------------- +%% Function: write(OProto, {Type, Data}) -> ok +%% +%% Type = {struct, StructDef} | +%% {list, Type} | +%% {map, KeyType, ValType} | +%% {set, Type} | +%% BaseType +%% +%% Data = +%% tuple() -- for struct +%% | list() -- for list +%% | dictionary() -- for map +%% | set() -- for set +%% | term() -- for base types +%% +%% Description: +%%-------------------------------------------------------------------- +write(Proto, {{struct, StructDef}, Data}) + when is_list(StructDef), is_tuple(Data), length(StructDef) == size(Data) -> + ok = write(Proto, #protocol_struct_begin{}), + ok = struct_write_loop(Proto, StructDef, tuple_to_list(Data)), + ok = write(Proto, struct_end), + ok; + +write(Proto, {{list, Type}, Data}) + when is_list(Data) -> + ok = write(Proto, + #protocol_list_begin{ + etype = term_to_typeid(Type), + size = length(Data) + }), + lists:foreach(fun(Elem) -> + ok = write(Proto, {Type, Elem}) + end, + Data), + ok = write(Proto, list_end), + ok; + +write(Proto, {{map, KeyType, ValType}, Data}) -> + DataList = dict:to_list(Data), + ok = write(Proto, + #protocol_map_begin{ + ktype = term_to_typeid(KeyType), + vtype = term_to_typeid(ValType), + size = length(DataList) + }), + lists:foreach(fun({KeyData, ValData}) -> + ok = write(Proto, {KeyType, KeyData}), + ok = write(Proto, {ValType, ValData}) + end, + DataList), + ok = write(Proto, map_end), + ok; + +write(Proto, {{set, Type}, Data}) -> + true = sets:is_set(Data), + DataList = sets:to_list(Data), + ok = write(Proto, + #protocol_set_begin{ + etype = term_to_typeid(Type), + size = length(DataList) + }), + lists:foreach(fun(Elem) -> + ok = write(Proto, {Type, Elem}) + end, + DataList), + ok = write(Proto, set_end), + ok; + +write(#protocol{module = Module, + data = ModuleData}, Data) -> + Module:write(ModuleData, Data). + + +struct_write_loop(Proto, [{Fid, Type} | RestStructDef], [Data | RestData]) -> + case Data of + undefined -> + % null fields are skipped in response + skip; + _ -> + ok = write(Proto, + #protocol_field_begin{ + type = term_to_typeid(Type), + id = Fid + }), + ok = write(Proto, {Type, Data}), + ok = write(Proto, field_end) + end, + struct_write_loop(Proto, RestStructDef, RestData); +struct_write_loop(Proto, [], []) -> + ok = write(Proto, field_stop), + ok. -- 2.17.1