From 59b479e59b110f9b8554e64a0f7bd6fa95a408d7 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Mon, 29 Jun 2015 15:59:50 -0700 Subject: [PATCH] godep: update gogo version --- Godeps/Godeps.json | 4 +- .../github.com/gogo/protobuf/proto/Makefile | 5 +- .../gogo/protobuf/proto/all_test.go | 96 ++++++- .../github.com/gogo/protobuf/proto/clone.go | 25 +- .../gogo/protobuf/proto/clone_test.go | 28 +- .../github.com/gogo/protobuf/proto/decode.go | 108 +++++++- .../gogo/protobuf/proto/decode_gogo.go | 45 ---- .../github.com/gogo/protobuf/proto/encode.go | 248 +++++++++++++++++- .../gogo/protobuf/proto/encode_gogo.go | 77 ++---- .../github.com/gogo/protobuf/proto/equal.go | 15 ++ .../gogo/protobuf/proto/equal_test.go | 27 +- .../gogo/protobuf/proto/extensions.go | 9 + .../gogo/protobuf/proto/extensions_test.go | 61 ++++- .../src/github.com/gogo/protobuf/proto/lib.go | 144 ++++++---- .../gogo/protobuf/proto/pointer_reflect.go | 97 ++++++- .../gogo/protobuf/proto/pointer_unsafe.go | 50 +++- .../protobuf/proto/pointer_unsafe_gogo.go | 58 ---- .../gogo/protobuf/proto/properties.go | 152 ++++++++++- .../gogo/protobuf/proto/properties_gogo.go | 47 ---- .../protobuf/proto/proto3_proto/proto3.pb.go | 122 +++++++++ .../protobuf/proto/proto3_proto/proto3.proto | 68 +++++ .../gogo/protobuf/proto/proto3_test.go | 125 +++++++++ .../gogo/protobuf/proto/size_test.go | 24 +- .../gogo/protobuf/proto/skip_gogo.go | 8 +- .../gogo/protobuf/proto/testdata/Makefile | 14 +- .../gogo/protobuf/proto/testdata/test.pb.go | 41 +++ .../gogo/protobuf/proto/testdata/test.proto | 7 + .../github.com/gogo/protobuf/proto/text.go | 96 ++++++- .../gogo/protobuf/proto/text_parser.go | 70 +++++ .../gogo/protobuf/proto/text_parser_test.go | 45 +++- .../gogo/protobuf/proto/text_test.go | 31 ++- 31 files changed, 1643 insertions(+), 304 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_proto/proto3.pb.go create mode 100644 Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_proto/proto3.proto create mode 100644 Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 2f36a7edb..9564492c7 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,6 +1,6 @@ { "ImportPath": "github.com/coreos/etcd", - "GoVersion": "go1.4.2", + "GoVersion": "go1.4.1", "Packages": [ "./..." ], @@ -43,7 +43,7 @@ }, { "ImportPath": "github.com/gogo/protobuf/proto", - "Rev": "bc946d07d1016848dfd2507f90f0859c9471681e" + "Rev": "64f27bf06efee53589314a6e5a4af34cdd85adf6" }, { "ImportPath": "github.com/golang/protobuf/proto", diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/Makefile b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/Makefile index 66bd2f299..ba28d87a3 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/Makefile +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/Makefile @@ -37,4 +37,7 @@ test: install generate-test-pbs generate-test-pbs: - make install && cd testdata && make + make install + make -C testdata + protoc-min-version --version="3.0.0" --proto_path=.:../../../../ proto3_proto/proto3.proto + make diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/all_test.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/all_test.go index 195c9c8c5..c805ae4fd 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/all_test.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/all_test.go @@ -44,8 +44,8 @@ import ( "testing" "time" - . "./testdata" . "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto" + . "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata" ) var globalO *Buffer @@ -1252,7 +1252,8 @@ func TestProto1RepeatedGroup(t *testing.T) { } o := old() - if err := o.Marshal(pb); err != ErrRepeatedHasNil { + err := o.Marshal(pb) + if err == nil || !strings.Contains(err.Error(), "repeated field Message has nil") { t.Fatalf("unexpected or no error when marshaling: %v", err) } } @@ -1441,6 +1442,17 @@ func TestSetDefaultsWithRepeatedSubMessage(t *testing.T) { } } +func TestSetDefaultWithRepeatedNonMessage(t *testing.T) { + m := &MyMessage{ + Pet: []string{"turtle", "wombat"}, + } + expected := Clone(m) + SetDefaults(m) + if !Equal(m, expected) { + t.Errorf("\n got %v\nwant %v", m, expected) + } +} + func TestMaximumTagNumber(t *testing.T) { m := &MaxTag{ LastField: String("natural goat essence"), @@ -1833,6 +1845,86 @@ func fuzzUnmarshal(t *testing.T, data []byte) { Unmarshal(data, pb) } +func TestMapFieldMarshal(t *testing.T) { + m := &MessageWithMap{ + NameMapping: map[int32]string{ + 1: "Rob", + 4: "Ian", + 8: "Dave", + }, + } + b, err := Marshal(m) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + // b should be the concatenation of these three byte sequences in some order. + parts := []string{ + "\n\a\b\x01\x12\x03Rob", + "\n\a\b\x04\x12\x03Ian", + "\n\b\b\x08\x12\x04Dave", + } + ok := false + for i := range parts { + for j := range parts { + if j == i { + continue + } + for k := range parts { + if k == i || k == j { + continue + } + try := parts[i] + parts[j] + parts[k] + if bytes.Equal(b, []byte(try)) { + ok = true + break + } + } + } + } + if !ok { + t.Fatalf("Incorrect Marshal output.\n got %q\nwant %q (or a permutation of that)", b, parts[0]+parts[1]+parts[2]) + } + t.Logf("FYI b: %q", b) + + (new(Buffer)).DebugPrint("Dump of b", b) +} + +func TestMapFieldRoundTrips(t *testing.T) { + m := &MessageWithMap{ + NameMapping: map[int32]string{ + 1: "Rob", + 4: "Ian", + 8: "Dave", + }, + MsgMapping: map[int64]*FloatingPoint{ + 0x7001: {F: Float64(2.0)}, + }, + ByteMapping: map[bool][]byte{ + false: []byte("that's not right!"), + true: []byte("aye, 'tis true!"), + }, + } + b, err := Marshal(m) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + t.Logf("FYI b: %q", b) + m2 := new(MessageWithMap) + if err := Unmarshal(b, m2); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + for _, pair := range [][2]interface{}{ + {m.NameMapping, m2.NameMapping}, + {m.MsgMapping, m2.MsgMapping}, + {m.ByteMapping, m2.ByteMapping}, + } { + if !reflect.DeepEqual(pair[0], pair[1]) { + t.Errorf("Map did not survive a round trip.\ninitial: %v\n final: %v", pair[0], pair[1]) + } + } +} + // Benchmarks func testMsg() *GoTest { diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/clone.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/clone.go index 8f9de86f7..ba3e94bb9 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/clone.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/clone.go @@ -29,7 +29,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Protocol buffer deep copy. +// Protocol buffer deep copy and merge. // TODO: MessageSet and RawMessage. package proto @@ -118,6 +118,29 @@ func mergeAny(out, in reflect.Value) { case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int32, reflect.Int64, reflect.String, reflect.Uint32, reflect.Uint64: out.Set(in) + case reflect.Map: + if in.Len() == 0 { + return + } + if out.IsNil() { + out.Set(reflect.MakeMap(in.Type())) + } + // For maps with value types of *T or []byte we need to deep copy each value. + elemKind := in.Type().Elem().Kind() + for _, key := range in.MapKeys() { + var val reflect.Value + switch elemKind { + case reflect.Ptr: + val = reflect.New(in.Type().Elem().Elem()) + mergeAny(val, in.MapIndex(key)) + case reflect.Slice: + val = in.MapIndex(key) + val = reflect.ValueOf(append([]byte{}, val.Bytes()...)) + default: + val = in.MapIndex(key) + } + out.SetMapIndex(key, val) + } case reflect.Ptr: if in.IsNil() { return diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/clone_test.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/clone_test.go index 59bbcc5cc..7115418a0 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/clone_test.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/clone_test.go @@ -35,8 +35,7 @@ import ( "testing" "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto" - - pb "./testdata" + pb "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata" ) var cloneTestMessage = &pb.MyMessage{ @@ -189,6 +188,31 @@ var mergeTests = []struct { dst: &pb.OtherMessage{Value: []byte("bar")}, want: &pb.OtherMessage{Value: []byte("foo")}, }, + { + src: &pb.MessageWithMap{ + NameMapping: map[int32]string{6: "Nigel"}, + MsgMapping: map[int64]*pb.FloatingPoint{ + 0x4001: {F: proto.Float64(2.0)}, + }, + ByteMapping: map[bool][]byte{true: []byte("wowsa")}, + }, + dst: &pb.MessageWithMap{ + NameMapping: map[int32]string{ + 6: "Bruce", // should be overwritten + 7: "Andrew", + }, + }, + want: &pb.MessageWithMap{ + NameMapping: map[int32]string{ + 6: "Nigel", + 7: "Andrew", + }, + MsgMapping: map[int64]*pb.FloatingPoint{ + 0x4001: {F: proto.Float64(2.0)}, + }, + ByteMapping: map[bool][]byte{true: []byte("wowsa")}, + }, + }, } func TestMerge(t *testing.T) { diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/decode.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/decode.go index ac0ea333b..8bba59414 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/decode.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/decode.go @@ -178,7 +178,7 @@ func (p *Buffer) DecodeZigzag32() (x uint64, err error) { func (p *Buffer) DecodeRawBytes(alloc bool) (buf []byte, err error) { n, err := p.DecodeVarint() if err != nil { - return + return nil, err } nb := int(n) @@ -470,6 +470,15 @@ func (o *Buffer) dec_bool(p *Properties, base structPointer) error { return nil } +func (o *Buffer) dec_proto3_bool(p *Properties, base structPointer) error { + u, err := p.valDec(o) + if err != nil { + return err + } + *structPointer_BoolVal(base, p.field) = u != 0 + return nil +} + // Decode an int32. func (o *Buffer) dec_int32(p *Properties, base structPointer) error { u, err := p.valDec(o) @@ -480,6 +489,15 @@ func (o *Buffer) dec_int32(p *Properties, base structPointer) error { return nil } +func (o *Buffer) dec_proto3_int32(p *Properties, base structPointer) error { + u, err := p.valDec(o) + if err != nil { + return err + } + word32Val_Set(structPointer_Word32Val(base, p.field), uint32(u)) + return nil +} + // Decode an int64. func (o *Buffer) dec_int64(p *Properties, base structPointer) error { u, err := p.valDec(o) @@ -490,15 +508,31 @@ func (o *Buffer) dec_int64(p *Properties, base structPointer) error { return nil } +func (o *Buffer) dec_proto3_int64(p *Properties, base structPointer) error { + u, err := p.valDec(o) + if err != nil { + return err + } + word64Val_Set(structPointer_Word64Val(base, p.field), o, u) + return nil +} + // Decode a string. func (o *Buffer) dec_string(p *Properties, base structPointer) error { s, err := o.DecodeStringBytes() if err != nil { return err } - sp := new(string) - *sp = s - *structPointer_String(base, p.field) = sp + *structPointer_String(base, p.field) = &s + return nil +} + +func (o *Buffer) dec_proto3_string(p *Properties, base structPointer) error { + s, err := o.DecodeStringBytes() + if err != nil { + return err + } + *structPointer_StringVal(base, p.field) = s return nil } @@ -637,6 +671,72 @@ func (o *Buffer) dec_slice_slice_byte(p *Properties, base structPointer) error { return nil } +// Decode a map field. +func (o *Buffer) dec_new_map(p *Properties, base structPointer) error { + raw, err := o.DecodeRawBytes(false) + if err != nil { + return err + } + oi := o.index // index at the end of this map entry + o.index -= len(raw) // move buffer back to start of map entry + + mptr := structPointer_Map(base, p.field, p.mtype) // *map[K]V + if mptr.Elem().IsNil() { + mptr.Elem().Set(reflect.MakeMap(mptr.Type().Elem())) + } + v := mptr.Elem() // map[K]V + + // Prepare addressable doubly-indirect placeholders for the key and value types. + // See enc_new_map for why. + keyptr := reflect.New(reflect.PtrTo(p.mtype.Key())).Elem() // addressable *K + keybase := toStructPointer(keyptr.Addr()) // **K + + var valbase structPointer + var valptr reflect.Value + switch p.mtype.Elem().Kind() { + case reflect.Slice: + // []byte + var dummy []byte + valptr = reflect.ValueOf(&dummy) // *[]byte + valbase = toStructPointer(valptr) // *[]byte + case reflect.Ptr: + // message; valptr is **Msg; need to allocate the intermediate pointer + valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V + valptr.Set(reflect.New(valptr.Type().Elem())) + valbase = toStructPointer(valptr) + default: + // everything else + valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V + valbase = toStructPointer(valptr.Addr()) // **V + } + + // Decode. + // This parses a restricted wire format, namely the encoding of a message + // with two fields. See enc_new_map for the format. + for o.index < oi { + // tagcode for key and value properties are always a single byte + // because they have tags 1 and 2. + tagcode := o.buf[o.index] + o.index++ + switch tagcode { + case p.mkeyprop.tagcode[0]: + if err := p.mkeyprop.dec(o, p.mkeyprop, keybase); err != nil { + return err + } + case p.mvalprop.tagcode[0]: + if err := p.mvalprop.dec(o, p.mvalprop, valbase); err != nil { + return err + } + default: + // TODO: Should we silently skip this instead? + return fmt.Errorf("proto: bad map data tag %d", raw[0]) + } + } + + v.SetMapIndex(keyptr.Elem(), valptr.Elem()) + return nil +} + // Decode a group. func (o *Buffer) dec_struct_group(p *Properties, base structPointer) error { bas := structPointer_GetStructPointer(base, p.field) diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/decode_gogo.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/decode_gogo.go index 96161adbe..6a77aad76 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/decode_gogo.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/decode_gogo.go @@ -30,51 +30,6 @@ import ( "reflect" ) -// Decode a reference to a bool pointer. -func (o *Buffer) dec_ref_bool(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - if len(o.bools) == 0 { - o.bools = make([]bool, boolPoolSize) - } - o.bools[0] = u != 0 - *structPointer_RefBool(base, p.field) = o.bools[0] - o.bools = o.bools[1:] - return nil -} - -// Decode a reference to an int32 pointer. -func (o *Buffer) dec_ref_int32(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - refWord32_Set(structPointer_RefWord32(base, p.field), o, uint32(u)) - return nil -} - -// Decode a reference to an int64 pointer. -func (o *Buffer) dec_ref_int64(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - refWord64_Set(structPointer_RefWord64(base, p.field), o, u) - return nil -} - -// Decode a reference to a string pointer. -func (o *Buffer) dec_ref_string(p *Properties, base structPointer) error { - s, err := o.DecodeStringBytes() - if err != nil { - return err - } - *structPointer_RefString(base, p.field) = s - return nil -} - // Decode a reference to a struct pointer. func (o *Buffer) dec_ref_struct_message(p *Properties, base structPointer) (err error) { raw, e := o.DecodeRawBytes(false) diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/encode.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/encode.go index cbe4242e0..d1abc3328 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/encode.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/encode.go @@ -60,9 +60,9 @@ func (e *RequiredNotSetError) Error() string { } var ( - // ErrRepeatedHasNil is the error returned if Marshal is called with + // errRepeatedHasNil is the error returned if Marshal is called with // a struct with a repeated field containing a nil element. - ErrRepeatedHasNil = errors.New("proto: repeated field has nil element") + errRepeatedHasNil = errors.New("proto: repeated field has nil element") // ErrNil is the error returned if Marshal is called with nil. ErrNil = errors.New("proto: Marshal called with nil") @@ -298,6 +298,16 @@ func (o *Buffer) enc_bool(p *Properties, base structPointer) error { return nil } +func (o *Buffer) enc_proto3_bool(p *Properties, base structPointer) error { + v := *structPointer_BoolVal(base, p.field) + if !v { + return ErrNil + } + o.buf = append(o.buf, p.tagcode...) + p.valEnc(o, 1) + return nil +} + func size_bool(p *Properties, base structPointer) int { v := *structPointer_Bool(base, p.field) if v == nil { @@ -306,6 +316,14 @@ func size_bool(p *Properties, base structPointer) int { return len(p.tagcode) + 1 // each bool takes exactly one byte } +func size_proto3_bool(p *Properties, base structPointer) int { + v := *structPointer_BoolVal(base, p.field) + if !v { + return 0 + } + return len(p.tagcode) + 1 // each bool takes exactly one byte +} + // Encode an int32. func (o *Buffer) enc_int32(p *Properties, base structPointer) error { v := structPointer_Word32(base, p.field) @@ -318,6 +336,17 @@ func (o *Buffer) enc_int32(p *Properties, base structPointer) error { return nil } +func (o *Buffer) enc_proto3_int32(p *Properties, base structPointer) error { + v := structPointer_Word32Val(base, p.field) + x := int32(word32Val_Get(v)) // permit sign extension to use full 64-bit range + if x == 0 { + return ErrNil + } + o.buf = append(o.buf, p.tagcode...) + p.valEnc(o, uint64(x)) + return nil +} + func size_int32(p *Properties, base structPointer) (n int) { v := structPointer_Word32(base, p.field) if word32_IsNil(v) { @@ -329,6 +358,17 @@ func size_int32(p *Properties, base structPointer) (n int) { return } +func size_proto3_int32(p *Properties, base structPointer) (n int) { + v := structPointer_Word32Val(base, p.field) + x := int32(word32Val_Get(v)) // permit sign extension to use full 64-bit range + if x == 0 { + return 0 + } + n += len(p.tagcode) + n += p.valSize(uint64(x)) + return +} + // Encode a uint32. // Exactly the same as int32, except for no sign extension. func (o *Buffer) enc_uint32(p *Properties, base structPointer) error { @@ -342,6 +382,17 @@ func (o *Buffer) enc_uint32(p *Properties, base structPointer) error { return nil } +func (o *Buffer) enc_proto3_uint32(p *Properties, base structPointer) error { + v := structPointer_Word32Val(base, p.field) + x := word32Val_Get(v) + if x == 0 { + return ErrNil + } + o.buf = append(o.buf, p.tagcode...) + p.valEnc(o, uint64(x)) + return nil +} + func size_uint32(p *Properties, base structPointer) (n int) { v := structPointer_Word32(base, p.field) if word32_IsNil(v) { @@ -353,6 +404,17 @@ func size_uint32(p *Properties, base structPointer) (n int) { return } +func size_proto3_uint32(p *Properties, base structPointer) (n int) { + v := structPointer_Word32Val(base, p.field) + x := word32Val_Get(v) + if x == 0 { + return 0 + } + n += len(p.tagcode) + n += p.valSize(uint64(x)) + return +} + // Encode an int64. func (o *Buffer) enc_int64(p *Properties, base structPointer) error { v := structPointer_Word64(base, p.field) @@ -365,6 +427,17 @@ func (o *Buffer) enc_int64(p *Properties, base structPointer) error { return nil } +func (o *Buffer) enc_proto3_int64(p *Properties, base structPointer) error { + v := structPointer_Word64Val(base, p.field) + x := word64Val_Get(v) + if x == 0 { + return ErrNil + } + o.buf = append(o.buf, p.tagcode...) + p.valEnc(o, x) + return nil +} + func size_int64(p *Properties, base structPointer) (n int) { v := structPointer_Word64(base, p.field) if word64_IsNil(v) { @@ -376,6 +449,17 @@ func size_int64(p *Properties, base structPointer) (n int) { return } +func size_proto3_int64(p *Properties, base structPointer) (n int) { + v := structPointer_Word64Val(base, p.field) + x := word64Val_Get(v) + if x == 0 { + return 0 + } + n += len(p.tagcode) + n += p.valSize(x) + return +} + // Encode a string. func (o *Buffer) enc_string(p *Properties, base structPointer) error { v := *structPointer_String(base, p.field) @@ -388,6 +472,16 @@ func (o *Buffer) enc_string(p *Properties, base structPointer) error { return nil } +func (o *Buffer) enc_proto3_string(p *Properties, base structPointer) error { + v := *structPointer_StringVal(base, p.field) + if v == "" { + return ErrNil + } + o.buf = append(o.buf, p.tagcode...) + o.EncodeStringBytes(v) + return nil +} + func size_string(p *Properties, base structPointer) (n int) { v := *structPointer_String(base, p.field) if v == nil { @@ -399,6 +493,16 @@ func size_string(p *Properties, base structPointer) (n int) { return } +func size_proto3_string(p *Properties, base structPointer) (n int) { + v := *structPointer_StringVal(base, p.field) + if v == "" { + return 0 + } + n += len(p.tagcode) + n += sizeStringBytes(v) + return +} + // All protocol buffer fields are nillable, but be careful. func isNil(v reflect.Value) bool { switch v.Kind() { @@ -551,6 +655,16 @@ func (o *Buffer) enc_slice_byte(p *Properties, base structPointer) error { return nil } +func (o *Buffer) enc_proto3_slice_byte(p *Properties, base structPointer) error { + s := *structPointer_Bytes(base, p.field) + if len(s) == 0 { + return ErrNil + } + o.buf = append(o.buf, p.tagcode...) + o.EncodeRawBytes(s) + return nil +} + func size_slice_byte(p *Properties, base structPointer) (n int) { s := *structPointer_Bytes(base, p.field) if s == nil { @@ -561,6 +675,16 @@ func size_slice_byte(p *Properties, base structPointer) (n int) { return } +func size_proto3_slice_byte(p *Properties, base structPointer) (n int) { + s := *structPointer_Bytes(base, p.field) + if len(s) == 0 { + return 0 + } + n += len(p.tagcode) + n += sizeRawBytes(s) + return +} + // Encode a slice of int32s ([]int32). func (o *Buffer) enc_slice_int32(p *Properties, base structPointer) error { s := structPointer_Word32Slice(base, p.field) @@ -815,7 +939,7 @@ func (o *Buffer) enc_slice_struct_message(p *Properties, base structPointer) err for i := 0; i < l; i++ { structp := s.Index(i) if structPointer_IsNil(structp) { - return ErrRepeatedHasNil + return errRepeatedHasNil } // Can the object marshal itself? @@ -834,7 +958,7 @@ func (o *Buffer) enc_slice_struct_message(p *Properties, base structPointer) err err := o.enc_len_struct(p.sprop, structp, &state) if err != nil && !state.shouldContinue(err, nil) { if err == ErrNil { - return ErrRepeatedHasNil + return errRepeatedHasNil } return err } @@ -877,7 +1001,7 @@ func (o *Buffer) enc_slice_struct_group(p *Properties, base structPointer) error for i := 0; i < l; i++ { b := s.Index(i) if structPointer_IsNil(b) { - return ErrRepeatedHasNil + return errRepeatedHasNil } o.EncodeVarint(uint64((p.Tag << 3) | WireStartGroup)) @@ -886,7 +1010,7 @@ func (o *Buffer) enc_slice_struct_group(p *Properties, base structPointer) error if err != nil && !state.shouldContinue(err, nil) { if err == ErrNil { - return ErrRepeatedHasNil + return errRepeatedHasNil } return err } @@ -945,6 +1069,108 @@ func size_map(p *Properties, base structPointer) int { return sizeExtensionMap(v) } +// Encode a map field. +func (o *Buffer) enc_new_map(p *Properties, base structPointer) error { + var state errorState // XXX: or do we need to plumb this through? + + /* + A map defined as + map map_field = N; + is encoded in the same way as + message MapFieldEntry { + key_type key = 1; + value_type value = 2; + } + repeated MapFieldEntry map_field = N; + */ + + v := structPointer_Map(base, p.field, p.mtype).Elem() // map[K]V + if v.Len() == 0 { + return nil + } + + keycopy, valcopy, keybase, valbase := mapEncodeScratch(p.mtype) + + enc := func() error { + if err := p.mkeyprop.enc(o, p.mkeyprop, keybase); err != nil { + return err + } + if err := p.mvalprop.enc(o, p.mvalprop, valbase); err != nil { + return err + } + return nil + } + + keys := v.MapKeys() + sort.Sort(mapKeys(keys)) + for _, key := range keys { + val := v.MapIndex(key) + + keycopy.Set(key) + valcopy.Set(val) + + o.buf = append(o.buf, p.tagcode...) + if err := o.enc_len_thing(enc, &state); err != nil { + return err + } + } + return nil +} + +func size_new_map(p *Properties, base structPointer) int { + v := structPointer_Map(base, p.field, p.mtype).Elem() // map[K]V + + keycopy, valcopy, keybase, valbase := mapEncodeScratch(p.mtype) + + n := 0 + for _, key := range v.MapKeys() { + val := v.MapIndex(key) + keycopy.Set(key) + valcopy.Set(val) + + // Tag codes for key and val are the responsibility of the sub-sizer. + keysize := p.mkeyprop.size(p.mkeyprop, keybase) + valsize := p.mvalprop.size(p.mvalprop, valbase) + entry := keysize + valsize + // Add on tag code and length of map entry itself. + n += len(p.tagcode) + sizeVarint(uint64(entry)) + entry + } + return n +} + +// mapEncodeScratch returns a new reflect.Value matching the map's value type, +// and a structPointer suitable for passing to an encoder or sizer. +func mapEncodeScratch(mapType reflect.Type) (keycopy, valcopy reflect.Value, keybase, valbase structPointer) { + // Prepare addressable doubly-indirect placeholders for the key and value types. + // This is needed because the element-type encoders expect **T, but the map iteration produces T. + + keycopy = reflect.New(mapType.Key()).Elem() // addressable K + keyptr := reflect.New(reflect.PtrTo(keycopy.Type())).Elem() // addressable *K + keyptr.Set(keycopy.Addr()) // + keybase = toStructPointer(keyptr.Addr()) // **K + + // Value types are more varied and require special handling. + switch mapType.Elem().Kind() { + case reflect.Slice: + // []byte + var dummy []byte + valcopy = reflect.ValueOf(&dummy).Elem() // addressable []byte + valbase = toStructPointer(valcopy.Addr()) + case reflect.Ptr: + // message; the generated field type is map[K]*Msg (so V is *Msg), + // so we only need one level of indirection. + valcopy = reflect.New(mapType.Elem()).Elem() // addressable V + valbase = toStructPointer(valcopy.Addr()) + default: + // everything else + valcopy = reflect.New(mapType.Elem()).Elem() // addressable V + valptr := reflect.New(reflect.PtrTo(valcopy.Type())).Elem() // addressable *V + valptr.Set(valcopy.Addr()) // + valbase = toStructPointer(valptr.Addr()) // **V + } + return +} + // Encode a struct. func (o *Buffer) enc_struct(prop *StructProperties, base structPointer) error { var state errorState @@ -960,6 +1186,9 @@ func (o *Buffer) enc_struct(prop *StructProperties, base structPointer) error { if p.Required && state.err == nil { state.err = &RequiredNotSetError{p.Name} } + } else if err == errRepeatedHasNil { + // Give more context to nil values in repeated fields. + return errors.New("repeated field " + p.OrigName + " has nil element") } else if !state.shouldContinue(err, p) { return err } @@ -999,10 +1228,15 @@ var zeroes [20]byte // longer than any conceivable sizeVarint // Encode a struct, preceded by its encoded length (as a varint). func (o *Buffer) enc_len_struct(prop *StructProperties, base structPointer, state *errorState) error { + return o.enc_len_thing(func() error { return o.enc_struct(prop, base) }, state) +} + +// Encode something, preceded by its encoded length (as a varint). +func (o *Buffer) enc_len_thing(enc func() error, state *errorState) error { iLen := len(o.buf) o.buf = append(o.buf, 0, 0, 0, 0) // reserve four bytes for length iMsg := len(o.buf) - err := o.enc_struct(prop, base) + err := enc() if err != nil && !state.shouldContinue(err, nil) { return err } diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/encode_gogo.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/encode_gogo.go index 66bc0bd53..f77cfb1ee 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/encode_gogo.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/encode_gogo.go @@ -40,6 +40,10 @@ import ( "reflect" ) +func NewRequiredNotSetError(field string) *RequiredNotSetError { + return &RequiredNotSetError{field} +} + type Sizer interface { Size() int } @@ -64,12 +68,9 @@ func size_ext_slice_byte(p *Properties, base structPointer) (n int) { // Encode a reference to bool pointer. func (o *Buffer) enc_ref_bool(p *Properties, base structPointer) error { - v := structPointer_RefBool(base, p.field) - if v == nil { - return ErrNil - } + v := *structPointer_BoolVal(base, p.field) x := 0 - if *v { + if v { x = 1 } o.buf = append(o.buf, p.tagcode...) @@ -78,53 +79,37 @@ func (o *Buffer) enc_ref_bool(p *Properties, base structPointer) error { } func size_ref_bool(p *Properties, base structPointer) int { - v := structPointer_RefBool(base, p.field) - if v == nil { - return 0 - } return len(p.tagcode) + 1 // each bool takes exactly one byte } // Encode a reference to int32 pointer. func (o *Buffer) enc_ref_int32(p *Properties, base structPointer) error { - v := structPointer_RefWord32(base, p.field) - if refWord32_IsNil(v) { - return ErrNil - } - x := int32(refWord32_Get(v)) + v := structPointer_Word32Val(base, p.field) + x := int32(word32Val_Get(v)) o.buf = append(o.buf, p.tagcode...) p.valEnc(o, uint64(x)) return nil } func size_ref_int32(p *Properties, base structPointer) (n int) { - v := structPointer_RefWord32(base, p.field) - if refWord32_IsNil(v) { - return 0 - } - x := int32(refWord32_Get(v)) + v := structPointer_Word32Val(base, p.field) + x := int32(word32Val_Get(v)) n += len(p.tagcode) n += p.valSize(uint64(x)) return } func (o *Buffer) enc_ref_uint32(p *Properties, base structPointer) error { - v := structPointer_RefWord32(base, p.field) - if refWord32_IsNil(v) { - return ErrNil - } - x := refWord32_Get(v) + v := structPointer_Word32Val(base, p.field) + x := word32Val_Get(v) o.buf = append(o.buf, p.tagcode...) p.valEnc(o, uint64(x)) return nil } func size_ref_uint32(p *Properties, base structPointer) (n int) { - v := structPointer_RefWord32(base, p.field) - if refWord32_IsNil(v) { - return 0 - } - x := refWord32_Get(v) + v := structPointer_Word32Val(base, p.field) + x := word32Val_Get(v) n += len(p.tagcode) n += p.valSize(uint64(x)) return @@ -132,22 +117,16 @@ func size_ref_uint32(p *Properties, base structPointer) (n int) { // Encode a reference to an int64 pointer. func (o *Buffer) enc_ref_int64(p *Properties, base structPointer) error { - v := structPointer_RefWord64(base, p.field) - if refWord64_IsNil(v) { - return ErrNil - } - x := refWord64_Get(v) + v := structPointer_Word64Val(base, p.field) + x := word64Val_Get(v) o.buf = append(o.buf, p.tagcode...) p.valEnc(o, x) return nil } func size_ref_int64(p *Properties, base structPointer) (n int) { - v := structPointer_RefWord64(base, p.field) - if refWord64_IsNil(v) { - return 0 - } - x := refWord64_Get(v) + v := structPointer_Word64Val(base, p.field) + x := word64Val_Get(v) n += len(p.tagcode) n += p.valSize(x) return @@ -155,24 +134,16 @@ func size_ref_int64(p *Properties, base structPointer) (n int) { // Encode a reference to a string pointer. func (o *Buffer) enc_ref_string(p *Properties, base structPointer) error { - v := structPointer_RefString(base, p.field) - if v == nil { - return ErrNil - } - x := *v + v := *structPointer_StringVal(base, p.field) o.buf = append(o.buf, p.tagcode...) - o.EncodeStringBytes(x) + o.EncodeStringBytes(v) return nil } func size_ref_string(p *Properties, base structPointer) (n int) { - v := structPointer_RefString(base, p.field) - if v == nil { - return 0 - } - x := *v + v := *structPointer_StringVal(base, p.field) n += len(p.tagcode) - n += sizeStringBytes(x) + n += sizeStringBytes(v) return } @@ -232,7 +203,7 @@ func (o *Buffer) enc_slice_ref_struct_message(p *Properties, base structPointer) for i := 0; i < l; i++ { structp := structPointer_Add(ss1, field(uintptr(i)*size)) if structPointer_IsNil(structp) { - return ErrRepeatedHasNil + return errRepeatedHasNil } // Can the object marshal itself? @@ -251,7 +222,7 @@ func (o *Buffer) enc_slice_ref_struct_message(p *Properties, base structPointer) err := o.enc_len_struct(p.sprop, structp, &state) if err != nil && !state.shouldContinue(err, nil) { if err == ErrNil { - return ErrRepeatedHasNil + return errRepeatedHasNil } return err } diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/equal.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/equal.go index ebdfdcaf7..d8673a3e9 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/equal.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/equal.go @@ -154,6 +154,21 @@ func equalAny(v1, v2 reflect.Value) bool { return v1.Float() == v2.Float() case reflect.Int32, reflect.Int64: return v1.Int() == v2.Int() + case reflect.Map: + if v1.Len() != v2.Len() { + return false + } + for _, key := range v1.MapKeys() { + val2 := v2.MapIndex(key) + if !val2.IsValid() { + // This key was not found in the second map. + return false + } + if !equalAny(v1.MapIndex(key), val2) { + return false + } + } + return true case reflect.Ptr: return equalAny(v1.Elem(), v2.Elem()) case reflect.Slice: diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/equal_test.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/equal_test.go index 1f99c518e..56b09e55d 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/equal_test.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/equal_test.go @@ -34,8 +34,8 @@ package proto_test import ( "testing" - pb "./testdata" . "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto" + pb "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata" ) // Four identical base messages. @@ -155,6 +155,31 @@ var EqualTests = []struct { }, true, }, + + { + "map same", + &pb.MessageWithMap{NameMapping: map[int32]string{1: "Ken"}}, + &pb.MessageWithMap{NameMapping: map[int32]string{1: "Ken"}}, + true, + }, + { + "map different entry", + &pb.MessageWithMap{NameMapping: map[int32]string{1: "Ken"}}, + &pb.MessageWithMap{NameMapping: map[int32]string{2: "Rob"}}, + false, + }, + { + "map different key only", + &pb.MessageWithMap{NameMapping: map[int32]string{1: "Ken"}}, + &pb.MessageWithMap{NameMapping: map[int32]string{2: "Ken"}}, + false, + }, + { + "map different value only", + &pb.MessageWithMap{NameMapping: map[int32]string{1: "Ken"}}, + &pb.MessageWithMap{NameMapping: map[int32]string{1: "Rob"}}, + false, + }, } func TestEqual(t *testing.T) { diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/extensions.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/extensions.go index fa1b98450..50e04e59f 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/extensions.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/extensions.go @@ -37,6 +37,7 @@ package proto import ( "errors" + "fmt" "reflect" "strconv" "sync" @@ -423,6 +424,14 @@ func SetExtension(pb extendableProto, extension *ExtensionDesc, value interface{ if typ != reflect.TypeOf(value) { return errors.New("proto: bad extension value type") } + // nil extension values need to be caught early, because the + // encoder can't distinguish an ErrNil due to a nil extension + // from an ErrNil due to a missing field. Extensions are + // always optional, so the encoder would just swallow the error + // and drop all the extensions from the encoded message. + if reflect.ValueOf(value).IsNil() { + return fmt.Errorf("proto: SetExtension called with nil value of type %T", value) + } return setExtension(pb, extension, value) } diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/extensions_test.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/extensions_test.go index a6a3f0e5c..275516093 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/extensions_test.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/extensions_test.go @@ -34,8 +34,8 @@ package proto_test import ( "testing" - pb "./testdata" "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto" + pb "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata" ) func TestGetExtensionsWithMissingExtensions(t *testing.T) { @@ -92,3 +92,62 @@ func TestGetExtensionStability(t *testing.T) { t.Errorf("GetExtension() not stable after unmarshaling") } } + +func TestExtensionsRoundTrip(t *testing.T) { + msg := &pb.MyMessage{} + ext1 := &pb.Ext{ + Data: proto.String("hi"), + } + ext2 := &pb.Ext{ + Data: proto.String("there"), + } + exists := proto.HasExtension(msg, pb.E_Ext_More) + if exists { + t.Error("Extension More present unexpectedly") + } + if err := proto.SetExtension(msg, pb.E_Ext_More, ext1); err != nil { + t.Error(err) + } + if err := proto.SetExtension(msg, pb.E_Ext_More, ext2); err != nil { + t.Error(err) + } + e, err := proto.GetExtension(msg, pb.E_Ext_More) + if err != nil { + t.Error(err) + } + x, ok := e.(*pb.Ext) + if !ok { + t.Errorf("e has type %T, expected testdata.Ext", e) + } else if *x.Data != "there" { + t.Errorf("SetExtension failed to overwrite, got %+v, not 'there'", x) + } + proto.ClearExtension(msg, pb.E_Ext_More) + if _, err = proto.GetExtension(msg, pb.E_Ext_More); err != proto.ErrMissingExtension { + t.Errorf("got %v, expected ErrMissingExtension", e) + } + if _, err := proto.GetExtension(msg, pb.E_X215); err == nil { + t.Error("expected bad extension error, got nil") + } + if err := proto.SetExtension(msg, pb.E_X215, 12); err == nil { + t.Error("expected extension err") + } + if err := proto.SetExtension(msg, pb.E_Ext_More, 12); err == nil { + t.Error("expected some sort of type mismatch error, got nil") + } +} + +func TestNilExtension(t *testing.T) { + msg := &pb.MyMessage{ + Count: proto.Int32(1), + } + if err := proto.SetExtension(msg, pb.E_Ext_Text, proto.String("hello")); err != nil { + t.Fatal(err) + } + if err := proto.SetExtension(msg, pb.E_Ext_More, (*pb.Ext)(nil)); err == nil { + t.Error("expected SetExtension to fail due to a nil extension") + } else if want := "proto: SetExtension called with nil value of type *testdata.Ext"; err.Error() != want { + t.Errorf("expected error %v, got %v", want, err) + } + // Note: if the behavior of Marshal is ever changed to ignore nil extensions, update + // this test to verify that E_Ext_Text is properly propagated through marshal->unmarshal. +} diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/lib.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/lib.go index 11dfccf6e..9e254e5ac 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/lib.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/lib.go @@ -39,7 +39,7 @@ - Names are turned from camel_case to CamelCase for export. - There are no methods on v to set fields; just treat - them as structure fields. + them as structure fields. - There are getters that return a field's value if set, and return the field's default value if unset. The getters work even if the receiver is a nil message. @@ -50,17 +50,16 @@ That is, optional or required field int32 f becomes F *int32. - Repeated fields are slices. - Helper functions are available to aid the setting of fields. - Helpers for getting values are superseded by the - GetFoo methods and their use is deprecated. - msg.Foo = proto.String("hello") // set field + msg.Foo = proto.String("hello") // set field - Constants are defined to hold the default values of all fields that have them. They have the form Default_StructName_FieldName. Because the getter methods handle defaulted values, direct use of these constants should be rare. - Enums are given type names and maps from names to values. - Enum values are prefixed with the enum's type name. Enum types have - a String method, and a Enum method to assist in message construction. - - Nested groups and enums have type names prefixed with the name of + Enum values are prefixed by the enclosing message's name, or by the + enum's type name if it is a top-level enum. Enum types have a String + method, and a Enum method to assist in message construction. + - Nested messages, groups and enums have type names prefixed with the name of the surrounding message type. - Extensions are given descriptor names that start with E_, followed by an underscore-delimited list of the nested messages @@ -74,7 +73,7 @@ package example; - enum FOO { X = 17; }; + enum FOO { X = 17; } message Test { required string label = 1; @@ -89,7 +88,8 @@ package example - import "github.com/gogo/protobuf/proto" + import proto "github.com/gogo/protobuf/proto" + import math "math" type FOO int32 const ( @@ -110,6 +110,14 @@ func (x FOO) String() string { return proto.EnumName(FOO_name, int32(x)) } + func (x *FOO) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(FOO_value, data) + if err != nil { + return err + } + *x = FOO(value) + return nil + } type Test struct { Label *string `protobuf:"bytes,1,req,name=label" json:"label,omitempty"` @@ -118,41 +126,41 @@ Optionalgroup *Test_OptionalGroup `protobuf:"group,4,opt,name=OptionalGroup" json:"optionalgroup,omitempty"` XXX_unrecognized []byte `json:"-"` } - func (this *Test) Reset() { *this = Test{} } - func (this *Test) String() string { return proto.CompactTextString(this) } + func (m *Test) Reset() { *m = Test{} } + func (m *Test) String() string { return proto.CompactTextString(m) } + func (*Test) ProtoMessage() {} const Default_Test_Type int32 = 77 - func (this *Test) GetLabel() string { - if this != nil && this.Label != nil { - return *this.Label + func (m *Test) GetLabel() string { + if m != nil && m.Label != nil { + return *m.Label } return "" } - func (this *Test) GetType() int32 { - if this != nil && this.Type != nil { - return *this.Type + func (m *Test) GetType() int32 { + if m != nil && m.Type != nil { + return *m.Type } return Default_Test_Type } - func (this *Test) GetOptionalgroup() *Test_OptionalGroup { - if this != nil { - return this.Optionalgroup + func (m *Test) GetOptionalgroup() *Test_OptionalGroup { + if m != nil { + return m.Optionalgroup } return nil } type Test_OptionalGroup struct { - RequiredField *string `protobuf:"bytes,5,req" json:"RequiredField,omitempty"` - XXX_unrecognized []byte `json:"-"` + RequiredField *string `protobuf:"bytes,5,req" json:"RequiredField,omitempty"` } - func (this *Test_OptionalGroup) Reset() { *this = Test_OptionalGroup{} } - func (this *Test_OptionalGroup) String() string { return proto.CompactTextString(this) } + func (m *Test_OptionalGroup) Reset() { *m = Test_OptionalGroup{} } + func (m *Test_OptionalGroup) String() string { return proto.CompactTextString(m) } - func (this *Test_OptionalGroup) GetRequiredField() string { - if this != nil && this.RequiredField != nil { - return *this.RequiredField + func (m *Test_OptionalGroup) GetRequiredField() string { + if m != nil && m.RequiredField != nil { + return *m.RequiredField } return "" } @@ -169,14 +177,14 @@ "log" "github.com/gogo/protobuf/proto" - "./example.pb" + pb "./example.pb" ) func main() { - test := &example.Test{ + test := &pb.Test{ Label: proto.String("hello"), Type: proto.Int32(17), - Optionalgroup: &example.Test_OptionalGroup{ + Optionalgroup: &pb.Test_OptionalGroup{ RequiredField: proto.String("good bye"), }, } @@ -184,7 +192,7 @@ if err != nil { log.Fatal("marshaling error: ", err) } - newTest := new(example.Test) + newTest := &pb.Test{} err = proto.Unmarshal(data, newTest) if err != nil { log.Fatal("unmarshaling error: ", err) @@ -323,9 +331,7 @@ func Float64(v float64) *float64 { // Uint32 is a helper routine that allocates a new uint32 value // to store v and returns a pointer to it. func Uint32(v uint32) *uint32 { - p := new(uint32) - *p = v - return p + return &v } // Uint64 is a helper routine that allocates a new uint64 value @@ -601,13 +607,15 @@ func setDefaults(v reflect.Value, recur, zeros bool) { for _, ni := range dm.nested { f := v.Field(ni) - if f.IsNil() { - continue - } - // f is *T or []*T - if f.Kind() == reflect.Ptr { + // f is *T or []*T or map[T]*T + switch f.Kind() { + case reflect.Ptr: + if f.IsNil() { + continue + } setDefaults(f, recur, zeros) - } else { + + case reflect.Slice: for i := 0; i < f.Len(); i++ { e := f.Index(i) if e.IsNil() { @@ -615,6 +623,15 @@ func setDefaults(v reflect.Value, recur, zeros bool) { } setDefaults(e, recur, zeros) } + + case reflect.Map: + for _, k := range f.MapKeys() { + e := f.MapIndex(k) + if e.IsNil() { + continue + } + setDefaults(e, recur, zeros) + } } } } @@ -640,10 +657,6 @@ type scalarField struct { value interface{} // the proto-declared default value, or nil } -func ptrToStruct(t reflect.Type) bool { - return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct -} - // t is a struct type. func buildDefaultMessage(t reflect.Type) (dm defaultMessage) { sprop := GetProperties(t) @@ -655,9 +668,33 @@ func buildDefaultMessage(t reflect.Type) (dm defaultMessage) { } ft := t.Field(fi).Type - // nested messages - if ptrToStruct(ft) || (ft.Kind() == reflect.Slice && ptrToStruct(ft.Elem())) { - dm.nested = append(dm.nested, fi) + var canHaveDefault, nestedMessage bool + switch ft.Kind() { + case reflect.Ptr: + if ft.Elem().Kind() == reflect.Struct { + nestedMessage = true + } else { + canHaveDefault = true // proto2 scalar field + } + + case reflect.Slice: + switch ft.Elem().Kind() { + case reflect.Ptr: + nestedMessage = true // repeated message + case reflect.Uint8: + canHaveDefault = true // bytes field + } + + case reflect.Map: + if ft.Elem().Kind() == reflect.Ptr { + nestedMessage = true // map with message values + } + } + + if !canHaveDefault { + if nestedMessage { + dm.nested = append(dm.nested, fi) + } continue } @@ -738,3 +775,16 @@ func buildDefaultMessage(t reflect.Type) (dm defaultMessage) { return dm } + +// Map fields may have key types of non-float scalars, strings and enums. +// The easiest way to sort them in some deterministic order is to use fmt. +// If this turns out to be inefficient we can always consider other options, +// such as doing a Schwartzian transform. + +type mapKeys []reflect.Value + +func (s mapKeys) Len() int { return len(s) } +func (s mapKeys) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s mapKeys) Less(i, j int) bool { + return fmt.Sprint(s[i].Interface()) < fmt.Sprint(s[j].Interface()) +} diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/pointer_reflect.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/pointer_reflect.go index 0c53417f0..c68b12525 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/pointer_reflect.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/pointer_reflect.go @@ -29,7 +29,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +build appengine,!appenginevm +// +build appengine // This file contains an implementation of proto field accesses using package reflect. // It is slower than the code in pointer_unsafe.go but it avoids package unsafe and can @@ -114,6 +114,11 @@ func structPointer_Bool(p structPointer, f field) **bool { return structPointer_ifield(p, f).(**bool) } +// BoolVal returns the address of a bool field in the struct. +func structPointer_BoolVal(p structPointer, f field) *bool { + return structPointer_ifield(p, f).(*bool) +} + // BoolSlice returns the address of a []bool field in the struct. func structPointer_BoolSlice(p structPointer, f field) *[]bool { return structPointer_ifield(p, f).(*[]bool) @@ -124,6 +129,11 @@ func structPointer_String(p structPointer, f field) **string { return structPointer_ifield(p, f).(**string) } +// StringVal returns the address of a string field in the struct. +func structPointer_StringVal(p structPointer, f field) *string { + return structPointer_ifield(p, f).(*string) +} + // StringSlice returns the address of a []string field in the struct. func structPointer_StringSlice(p structPointer, f field) *[]string { return structPointer_ifield(p, f).(*[]string) @@ -134,6 +144,11 @@ func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension { return structPointer_ifield(p, f).(*map[int32]Extension) } +// Map returns the reflect.Value for the address of a map field in the struct. +func structPointer_Map(p structPointer, f field, typ reflect.Type) reflect.Value { + return structPointer_field(p, f).Addr() +} + // SetStructPointer writes a *struct field in the struct. func structPointer_SetStructPointer(p structPointer, f field, q structPointer) { structPointer_field(p, f).Set(q.v) @@ -235,6 +250,49 @@ func structPointer_Word32(p structPointer, f field) word32 { return word32{structPointer_field(p, f)} } +// A word32Val represents a field of type int32, uint32, float32, or enum. +// That is, v.Type() is int32, uint32, float32, or enum and v is assignable. +type word32Val struct { + v reflect.Value +} + +// Set sets *p to x. +func word32Val_Set(p word32Val, x uint32) { + switch p.v.Type() { + case int32Type: + p.v.SetInt(int64(x)) + return + case uint32Type: + p.v.SetUint(uint64(x)) + return + case float32Type: + p.v.SetFloat(float64(math.Float32frombits(x))) + return + } + + // must be enum + p.v.SetInt(int64(int32(x))) +} + +// Get gets the bits pointed at by p, as a uint32. +func word32Val_Get(p word32Val) uint32 { + elem := p.v + switch elem.Kind() { + case reflect.Int32: + return uint32(elem.Int()) + case reflect.Uint32: + return uint32(elem.Uint()) + case reflect.Float32: + return math.Float32bits(float32(elem.Float())) + } + panic("unreachable") +} + +// Word32Val returns a reference to a int32, uint32, float32, or enum field in the struct. +func structPointer_Word32Val(p structPointer, f field) word32Val { + return word32Val{structPointer_field(p, f)} +} + // A word32Slice is a slice of 32-bit values. // That is, v.Type() is []int32, []uint32, []float32, or []enum. type word32Slice struct { @@ -339,6 +397,43 @@ func structPointer_Word64(p structPointer, f field) word64 { return word64{structPointer_field(p, f)} } +// word64Val is like word32Val but for 64-bit values. +type word64Val struct { + v reflect.Value +} + +func word64Val_Set(p word64Val, o *Buffer, x uint64) { + switch p.v.Type() { + case int64Type: + p.v.SetInt(int64(x)) + return + case uint64Type: + p.v.SetUint(x) + return + case float64Type: + p.v.SetFloat(math.Float64frombits(x)) + return + } + panic("unreachable") +} + +func word64Val_Get(p word64Val) uint64 { + elem := p.v + switch elem.Kind() { + case reflect.Int64: + return uint64(elem.Int()) + case reflect.Uint64: + return elem.Uint() + case reflect.Float64: + return math.Float64bits(elem.Float()) + } + panic("unreachable") +} + +func structPointer_Word64Val(p structPointer, f field) word64Val { + return word64Val{structPointer_field(p, f)} +} + type word64Slice struct { v reflect.Value } diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/pointer_unsafe.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/pointer_unsafe.go index 7f3473e2f..48bc0fa48 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/pointer_unsafe.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/pointer_unsafe.go @@ -29,7 +29,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +build !appengine appenginevm +// +build !appengine // This file contains the implementation of the proto field accesses using package unsafe. @@ -100,6 +100,11 @@ func structPointer_Bool(p structPointer, f field) **bool { return (**bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) } +// BoolVal returns the address of a bool field in the struct. +func structPointer_BoolVal(p structPointer, f field) *bool { + return (*bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) +} + // BoolSlice returns the address of a []bool field in the struct. func structPointer_BoolSlice(p structPointer, f field) *[]bool { return (*[]bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) @@ -110,6 +115,11 @@ func structPointer_String(p structPointer, f field) **string { return (**string)(unsafe.Pointer(uintptr(p) + uintptr(f))) } +// StringVal returns the address of a string field in the struct. +func structPointer_StringVal(p structPointer, f field) *string { + return (*string)(unsafe.Pointer(uintptr(p) + uintptr(f))) +} + // StringSlice returns the address of a []string field in the struct. func structPointer_StringSlice(p structPointer, f field) *[]string { return (*[]string)(unsafe.Pointer(uintptr(p) + uintptr(f))) @@ -120,6 +130,11 @@ func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension { return (*map[int32]Extension)(unsafe.Pointer(uintptr(p) + uintptr(f))) } +// Map returns the reflect.Value for the address of a map field in the struct. +func structPointer_Map(p structPointer, f field, typ reflect.Type) reflect.Value { + return reflect.NewAt(typ, unsafe.Pointer(uintptr(p)+uintptr(f))) +} + // SetStructPointer writes a *struct field in the struct. func structPointer_SetStructPointer(p structPointer, f field, q structPointer) { *(*structPointer)(unsafe.Pointer(uintptr(p) + uintptr(f))) = q @@ -170,6 +185,24 @@ func structPointer_Word32(p structPointer, f field) word32 { return word32((**uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) } +// A word32Val is the address of a 32-bit value field. +type word32Val *uint32 + +// Set sets *p to x. +func word32Val_Set(p word32Val, x uint32) { + *p = x +} + +// Get gets the value pointed at by p. +func word32Val_Get(p word32Val) uint32 { + return *p +} + +// Word32Val returns the address of a *int32, *uint32, *float32, or *enum field in the struct. +func structPointer_Word32Val(p structPointer, f field) word32Val { + return word32Val((*uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) +} + // A word32Slice is a slice of 32-bit values. type word32Slice []uint32 @@ -206,6 +239,21 @@ func structPointer_Word64(p structPointer, f field) word64 { return word64((**uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) } +// word64Val is like word32Val but for 64-bit values. +type word64Val *uint64 + +func word64Val_Set(p word64Val, o *Buffer, x uint64) { + *p = x +} + +func word64Val_Get(p word64Val) uint64 { + return *p +} + +func structPointer_Word64Val(p structPointer, f field) word64Val { + return word64Val((*uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) +} + // word64Slice is like word32Slice but for 64-bit values. type word64Slice []uint64 diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/pointer_unsafe_gogo.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/pointer_unsafe_gogo.go index 797a6118f..6bc85fa98 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/pointer_unsafe_gogo.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/pointer_unsafe_gogo.go @@ -87,16 +87,6 @@ func appendStructPointer(base structPointer, f field, typ reflect.Type) structPo return structPointer(unsafe.Pointer(uintptr(unsafe.Pointer(bas)) + uintptr(uintptr(newLen-1)*size))) } -// RefBool returns a *bool field in the struct. -func structPointer_RefBool(p structPointer, f field) *bool { - return (*bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - -// RefString returns the address of a string field in the struct. -func structPointer_RefString(p structPointer, f field) *string { - return (*string)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - func structPointer_FieldPointer(p structPointer, f field) structPointer { return structPointer(unsafe.Pointer(uintptr(p) + uintptr(f))) } @@ -116,51 +106,3 @@ func structPointer_Add(p structPointer, size field) structPointer { func structPointer_Len(p structPointer, f field) int { return len(*(*[]interface{})(unsafe.Pointer(structPointer_GetRefStructPointer(p, f)))) } - -// refWord32 is the address of a 32-bit value field. -type refWord32 *uint32 - -func refWord32_IsNil(p refWord32) bool { - return p == nil -} - -func refWord32_Set(p refWord32, o *Buffer, x uint32) { - if len(o.uint32s) == 0 { - o.uint32s = make([]uint32, uint32PoolSize) - } - o.uint32s[0] = x - *p = o.uint32s[0] - o.uint32s = o.uint32s[1:] -} - -func refWord32_Get(p refWord32) uint32 { - return *p -} - -func structPointer_RefWord32(p structPointer, f field) refWord32 { - return refWord32((*uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) -} - -// refWord64 is like refWord32 but for 32-bit values. -type refWord64 *uint64 - -func refWord64_Set(p refWord64, o *Buffer, x uint64) { - if len(o.uint64s) == 0 { - o.uint64s = make([]uint64, uint64PoolSize) - } - o.uint64s[0] = x - *p = o.uint64s[0] - o.uint64s = o.uint64s[1:] -} - -func refWord64_IsNil(p refWord64) bool { - return p == nil -} - -func refWord64_Get(p refWord64) uint64 { - return *p -} - -func structPointer_RefWord64(p structPointer, f field) refWord64 { - return refWord64((*uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) -} diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/properties.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/properties.go index f5a2a8b0b..13245c00d 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/properties.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/properties.go @@ -160,6 +160,7 @@ type Properties struct { Repeated bool Packed bool // relevant for repeated primitives only Enum string // set for enum types only + proto3 bool // whether this is known to be a proto3 field; set for []byte only Default string // default value HasDefault bool // whether an explicit default was provided @@ -178,6 +179,10 @@ type Properties struct { isMarshaler bool isUnmarshaler bool + mtype reflect.Type // set for map types only + mkeyprop *Properties // set for map types only + mvalprop *Properties // set for map types only + size sizer valSize valueSizer // set for bool and numeric types only @@ -208,6 +213,9 @@ func (p *Properties) String() string { if p.OrigName != p.Name { s += ",name=" + p.OrigName } + if p.proto3 { + s += ",proto3" + } if len(p.Enum) > 0 { s += ",enum=" + p.Enum } @@ -282,6 +290,8 @@ func (p *Properties) Parse(s string) { p.OrigName = f[5:] case strings.HasPrefix(f, "enum="): p.Enum = f[5:] + case f == "proto3": + p.proto3 = true case strings.HasPrefix(f, "def="): p.HasDefault = true p.Default = f[4:] // rest of string @@ -305,7 +315,7 @@ func logNoSliceEnc(t1, t2 reflect.Type) { var protoMessageType = reflect.TypeOf((*Message)(nil)).Elem() // Initialize the fields for encoding and decoding. -func (p *Properties) setEncAndDec(typ reflect.Type, lockGetProp bool) { +func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lockGetProp bool) { p.enc = nil p.dec = nil p.size = nil @@ -316,13 +326,96 @@ func (p *Properties) setEncAndDec(typ reflect.Type, lockGetProp bool) { } switch t1 := typ; t1.Kind() { default: - if !p.setNonNullableEncAndDec(t1) { - fmt.Fprintf(os.Stderr, "proto: no coders for %v\n", t1) + fmt.Fprintf(os.Stderr, "proto: no coders for %v\n", t1) + + // proto3 scalar types + + case reflect.Bool: + if p.proto3 { + p.enc = (*Buffer).enc_proto3_bool + p.dec = (*Buffer).dec_proto3_bool + p.size = size_proto3_bool + } else { + p.enc = (*Buffer).enc_ref_bool + p.dec = (*Buffer).dec_proto3_bool + p.size = size_ref_bool } + case reflect.Int32: + if p.proto3 { + p.enc = (*Buffer).enc_proto3_int32 + p.dec = (*Buffer).dec_proto3_int32 + p.size = size_proto3_int32 + } else { + p.enc = (*Buffer).enc_ref_int32 + p.dec = (*Buffer).dec_proto3_int32 + p.size = size_ref_int32 + } + case reflect.Uint32: + if p.proto3 { + p.enc = (*Buffer).enc_proto3_uint32 + p.dec = (*Buffer).dec_proto3_int32 // can reuse + p.size = size_proto3_uint32 + } else { + p.enc = (*Buffer).enc_ref_uint32 + p.dec = (*Buffer).dec_proto3_int32 // can reuse + p.size = size_ref_uint32 + } + case reflect.Int64, reflect.Uint64: + if p.proto3 { + p.enc = (*Buffer).enc_proto3_int64 + p.dec = (*Buffer).dec_proto3_int64 + p.size = size_proto3_int64 + } else { + p.enc = (*Buffer).enc_ref_int64 + p.dec = (*Buffer).dec_proto3_int64 + p.size = size_ref_int64 + } + case reflect.Float32: + if p.proto3 { + p.enc = (*Buffer).enc_proto3_uint32 // can just treat them as bits + p.dec = (*Buffer).dec_proto3_int32 + p.size = size_proto3_uint32 + } else { + p.enc = (*Buffer).enc_ref_uint32 // can just treat them as bits + p.dec = (*Buffer).dec_proto3_int32 + p.size = size_ref_uint32 + } + case reflect.Float64: + if p.proto3 { + p.enc = (*Buffer).enc_proto3_int64 // can just treat them as bits + p.dec = (*Buffer).dec_proto3_int64 + p.size = size_proto3_int64 + } else { + p.enc = (*Buffer).enc_ref_int64 // can just treat them as bits + p.dec = (*Buffer).dec_proto3_int64 + p.size = size_ref_int64 + } + case reflect.String: + if p.proto3 { + p.enc = (*Buffer).enc_proto3_string + p.dec = (*Buffer).dec_proto3_string + p.size = size_proto3_string + } else { + p.enc = (*Buffer).enc_ref_string + p.dec = (*Buffer).dec_proto3_string + p.size = size_ref_string + } + case reflect.Struct: + p.stype = typ + p.isMarshaler = isMarshaler(typ) + p.isUnmarshaler = isUnmarshaler(typ) + if p.Wire == "bytes" { + p.enc = (*Buffer).enc_ref_struct_message + p.dec = (*Buffer).dec_ref_struct_message + p.size = size_ref_struct_message + } else { + fmt.Fprintf(os.Stderr, "proto: no coders for struct %T\n", typ) + } + case reflect.Ptr: switch t2 := t1.Elem(); t2.Kind() { default: - fmt.Fprintf(os.Stderr, "proto: no encoder function for %T -> %T\n", t1, t2) + fmt.Fprintf(os.Stderr, "proto: no encoder function for %v -> %v\n", t1, t2) break case reflect.Bool: p.enc = (*Buffer).enc_bool @@ -416,6 +509,15 @@ func (p *Properties) setEncAndDec(typ reflect.Type, lockGetProp bool) { p.enc = (*Buffer).enc_slice_byte p.dec = (*Buffer).dec_slice_byte p.size = size_slice_byte + // This is a []byte, which is either a bytes field, + // or the value of a map field. In the latter case, + // we always encode an empty []byte, so we should not + // use the proto3 enc/size funcs. + // f == nil iff this is the key/value of a map field. + if p.proto3 && f != nil { + p.enc = (*Buffer).enc_proto3_slice_byte + p.size = size_proto3_slice_byte + } case reflect.Float32, reflect.Float64: switch t2.Bits() { case 32: @@ -480,6 +582,23 @@ func (p *Properties) setEncAndDec(typ reflect.Type, lockGetProp bool) { case reflect.Struct: p.setSliceOfNonPointerStructs(t1) } + + case reflect.Map: + p.enc = (*Buffer).enc_new_map + p.dec = (*Buffer).dec_new_map + p.size = size_new_map + + p.mtype = t1 + p.mkeyprop = &Properties{} + p.mkeyprop.init(reflect.PtrTo(p.mtype.Key()), "Key", f.Tag.Get("protobuf_key"), nil, lockGetProp) + p.mvalprop = &Properties{} + vtype := p.mtype.Elem() + if vtype.Kind() != reflect.Ptr && vtype.Kind() != reflect.Slice { + // The value type is not a message (*T) or bytes ([]byte), + // so we need encoders for the pointer to this type. + vtype = reflect.PtrTo(vtype) + } + p.mvalprop.init(vtype, "Value", f.Tag.Get("protobuf_val"), nil, lockGetProp) } p.setTag(lockGetProp) } @@ -539,11 +658,11 @@ func (p *Properties) init(typ reflect.Type, name, tag string, f *reflect.StructF return } p.Parse(tag) - p.setEncAndDec(typ, lockGetProp) + p.setEncAndDec(typ, f, lockGetProp) } var ( - mutex sync.Mutex + propertiesMu sync.RWMutex propertiesMap = make(map[reflect.Type]*StructProperties) ) @@ -553,13 +672,26 @@ func GetProperties(t reflect.Type) *StructProperties { if t.Kind() != reflect.Struct { panic("proto: type must have kind struct") } - mutex.Lock() - sprop := getPropertiesLocked(t) - mutex.Unlock() + + // Most calls to GetProperties in a long-running program will be + // retrieving details for types we have seen before. + propertiesMu.RLock() + sprop, ok := propertiesMap[t] + propertiesMu.RUnlock() + if ok { + if collectStats { + stats.Chit++ + } + return sprop + } + + propertiesMu.Lock() + sprop = getPropertiesLocked(t) + propertiesMu.Unlock() return sprop } -// getPropertiesLocked requires that mutex is held. +// getPropertiesLocked requires that propertiesMu is held. func getPropertiesLocked(t reflect.Type) *StructProperties { if prop, ok := propertiesMap[t]; ok { if collectStats { diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/properties_gogo.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/properties_gogo.go index 3518b7783..8daf9f776 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/properties_gogo.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/properties_gogo.go @@ -49,53 +49,6 @@ func (p *Properties) setCustomEncAndDec(typ reflect.Type) { } } -func (p *Properties) setNonNullableEncAndDec(typ reflect.Type) bool { - switch typ.Kind() { - case reflect.Bool: - p.enc = (*Buffer).enc_ref_bool - p.dec = (*Buffer).dec_ref_bool - p.size = size_ref_bool - case reflect.Int32: - p.enc = (*Buffer).enc_ref_int32 - p.dec = (*Buffer).dec_ref_int32 - p.size = size_ref_int32 - case reflect.Uint32: - p.enc = (*Buffer).enc_ref_uint32 - p.dec = (*Buffer).dec_ref_int32 - p.size = size_ref_uint32 - case reflect.Int64, reflect.Uint64: - p.enc = (*Buffer).enc_ref_int64 - p.dec = (*Buffer).dec_ref_int64 - p.size = size_ref_int64 - case reflect.Float32: - p.enc = (*Buffer).enc_ref_uint32 // can just treat them as bits - p.dec = (*Buffer).dec_ref_int32 - p.size = size_ref_uint32 - case reflect.Float64: - p.enc = (*Buffer).enc_ref_int64 // can just treat them as bits - p.dec = (*Buffer).dec_ref_int64 - p.size = size_ref_int64 - case reflect.String: - p.dec = (*Buffer).dec_ref_string - p.enc = (*Buffer).enc_ref_string - p.size = size_ref_string - case reflect.Struct: - p.stype = typ - p.isMarshaler = isMarshaler(typ) - p.isUnmarshaler = isUnmarshaler(typ) - if p.Wire == "bytes" { - p.enc = (*Buffer).enc_ref_struct_message - p.dec = (*Buffer).dec_ref_struct_message - p.size = size_ref_struct_message - } else { - fmt.Fprintf(os.Stderr, "proto: no coders for struct %T\n", typ) - } - default: - return false - } - return true -} - func (p *Properties) setSliceOfNonPointerStructs(typ reflect.Type) { t2 := typ.Elem() p.sstype = typ diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_proto/proto3.pb.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_proto/proto3.pb.go new file mode 100644 index 000000000..5ec11613b --- /dev/null +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_proto/proto3.pb.go @@ -0,0 +1,122 @@ +// Code generated by protoc-gen-gogo. +// source: proto3_proto/proto3.proto +// DO NOT EDIT! + +/* +Package proto3_proto is a generated protocol buffer package. + +It is generated from these files: + proto3_proto/proto3.proto + +It has these top-level messages: + Message + Nested + MessageWithMap +*/ +package proto3_proto + +import proto "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto" +import testdata "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal + +type Message_Humour int32 + +const ( + Message_UNKNOWN Message_Humour = 0 + Message_PUNS Message_Humour = 1 + Message_SLAPSTICK Message_Humour = 2 + Message_BILL_BAILEY Message_Humour = 3 +) + +var Message_Humour_name = map[int32]string{ + 0: "UNKNOWN", + 1: "PUNS", + 2: "SLAPSTICK", + 3: "BILL_BAILEY", +} +var Message_Humour_value = map[string]int32{ + "UNKNOWN": 0, + "PUNS": 1, + "SLAPSTICK": 2, + "BILL_BAILEY": 3, +} + +func (x Message_Humour) String() string { + return proto.EnumName(Message_Humour_name, int32(x)) +} + +type Message struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Hilarity Message_Humour `protobuf:"varint,2,opt,name=hilarity,proto3,enum=proto3_proto.Message_Humour" json:"hilarity,omitempty"` + HeightInCm uint32 `protobuf:"varint,3,opt,name=height_in_cm,proto3" json:"height_in_cm,omitempty"` + Data []byte `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"` + ResultCount int64 `protobuf:"varint,7,opt,name=result_count,proto3" json:"result_count,omitempty"` + TrueScotsman bool `protobuf:"varint,8,opt,name=true_scotsman,proto3" json:"true_scotsman,omitempty"` + Score float32 `protobuf:"fixed32,9,opt,name=score,proto3" json:"score,omitempty"` + Key []uint64 `protobuf:"varint,5,rep,name=key" json:"key,omitempty"` + Nested *Nested `protobuf:"bytes,6,opt,name=nested" json:"nested,omitempty"` + Terrain map[string]*Nested `protobuf:"bytes,10,rep,name=terrain" json:"terrain,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` + Proto2Field *testdata.SubDefaults `protobuf:"bytes,11,opt,name=proto2_field" json:"proto2_field,omitempty"` + Proto2Value map[string]*testdata.SubDefaults `protobuf:"bytes,13,rep,name=proto2_value" json:"proto2_value,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` +} + +func (m *Message) Reset() { *m = Message{} } +func (m *Message) String() string { return proto.CompactTextString(m) } +func (*Message) ProtoMessage() {} + +func (m *Message) GetNested() *Nested { + if m != nil { + return m.Nested + } + return nil +} + +func (m *Message) GetTerrain() map[string]*Nested { + if m != nil { + return m.Terrain + } + return nil +} + +func (m *Message) GetProto2Field() *testdata.SubDefaults { + if m != nil { + return m.Proto2Field + } + return nil +} + +func (m *Message) GetProto2Value() map[string]*testdata.SubDefaults { + if m != nil { + return m.Proto2Value + } + return nil +} + +type Nested struct { + Bunny string `protobuf:"bytes,1,opt,name=bunny,proto3" json:"bunny,omitempty"` +} + +func (m *Nested) Reset() { *m = Nested{} } +func (m *Nested) String() string { return proto.CompactTextString(m) } +func (*Nested) ProtoMessage() {} + +type MessageWithMap struct { + ByteMapping map[bool][]byte `protobuf:"bytes,1,rep,name=byte_mapping" json:"byte_mapping,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (m *MessageWithMap) Reset() { *m = MessageWithMap{} } +func (m *MessageWithMap) String() string { return proto.CompactTextString(m) } +func (*MessageWithMap) ProtoMessage() {} + +func (m *MessageWithMap) GetByteMapping() map[bool][]byte { + if m != nil { + return m.ByteMapping + } + return nil +} + +func init() { + proto.RegisterEnum("proto3_proto.Message_Humour", Message_Humour_name, Message_Humour_value) +} diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_proto/proto3.proto b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_proto/proto3.proto new file mode 100644 index 000000000..ca670015a --- /dev/null +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_proto/proto3.proto @@ -0,0 +1,68 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2014 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package proto3_proto; + +import "github.com/gogo/protobuf/proto/testdata/test.proto"; + +message Message { + enum Humour { + UNKNOWN = 0; + PUNS = 1; + SLAPSTICK = 2; + BILL_BAILEY = 3; + } + + string name = 1; + Humour hilarity = 2; + uint32 height_in_cm = 3; + bytes data = 4; + int64 result_count = 7; + bool true_scotsman = 8; + float score = 9; + + repeated uint64 key = 5; + Nested nested = 6; + + map terrain = 10; + testdata.SubDefaults proto2_field = 11; + map proto2_value = 13; +} + +message Nested { + string bunny = 1; +} + +message MessageWithMap { + map byte_mapping = 1; +} diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_test.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_test.go new file mode 100644 index 000000000..27e838727 --- /dev/null +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_test.go @@ -0,0 +1,125 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2014 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto_test + +import ( + "testing" + + "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto" + pb "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_proto" + tpb "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata" +) + +func TestProto3ZeroValues(t *testing.T) { + tests := []struct { + desc string + m proto.Message + }{ + {"zero message", &pb.Message{}}, + {"empty bytes field", &pb.Message{Data: []byte{}}}, + } + for _, test := range tests { + b, err := proto.Marshal(test.m) + if err != nil { + t.Errorf("%s: proto.Marshal: %v", test.desc, err) + continue + } + if len(b) > 0 { + t.Errorf("%s: Encoding is non-empty: %q", test.desc, b) + } + } +} + +func TestRoundTripProto3(t *testing.T) { + m := &pb.Message{ + Name: "David", // (2 | 1<<3): 0x0a 0x05 "David" + Hilarity: pb.Message_PUNS, // (0 | 2<<3): 0x10 0x01 + HeightInCm: 178, // (0 | 3<<3): 0x18 0xb2 0x01 + Data: []byte("roboto"), // (2 | 4<<3): 0x20 0x06 "roboto" + ResultCount: 47, // (0 | 7<<3): 0x38 0x2f + TrueScotsman: true, // (0 | 8<<3): 0x40 0x01 + Score: 8.1, // (5 | 9<<3): 0x4d <8.1> + + Key: []uint64{1, 0xdeadbeef}, + Nested: &pb.Nested{ + Bunny: "Monty", + }, + } + t.Logf(" m: %v", m) + + b, err := proto.Marshal(m) + if err != nil { + t.Fatalf("proto.Marshal: %v", err) + } + t.Logf(" b: %q", b) + + m2 := new(pb.Message) + if err := proto.Unmarshal(b, m2); err != nil { + t.Fatalf("proto.Unmarshal: %v", err) + } + t.Logf("m2: %v", m2) + + if !proto.Equal(m, m2) { + t.Errorf("proto.Equal returned false:\n m: %v\nm2: %v", m, m2) + } +} + +func TestProto3SetDefaults(t *testing.T) { + in := &pb.Message{ + Terrain: map[string]*pb.Nested{ + "meadow": new(pb.Nested), + }, + Proto2Field: new(tpb.SubDefaults), + Proto2Value: map[string]*tpb.SubDefaults{ + "badlands": new(tpb.SubDefaults), + }, + } + + got := proto.Clone(in).(*pb.Message) + proto.SetDefaults(got) + + // There are no defaults in proto3. Everything should be the zero value, but + // we need to remember to set defaults for nested proto2 messages. + want := &pb.Message{ + Terrain: map[string]*pb.Nested{ + "meadow": new(pb.Nested), + }, + Proto2Field: &tpb.SubDefaults{N: proto.Int64(7)}, + Proto2Value: map[string]*tpb.SubDefaults{ + "badlands": {N: proto.Int64(7)}, + }, + } + + if !proto.Equal(got, want) { + t.Errorf("with in = %v\nproto.SetDefaults(in) =>\ngot %v\nwant %v", in, got, want) + } +} diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/size_test.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/size_test.go index fe0dfddd2..4fbec925b 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/size_test.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/size_test.go @@ -33,10 +33,12 @@ package proto_test import ( "log" + "strings" "testing" - pb "./testdata" . "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto" + proto3pb "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_proto" + pb "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata" ) var messageWithExtension1 = &pb.MyMessage{Count: Int32(7)} @@ -102,6 +104,26 @@ var SizeTests = []struct { {"unrecognized", &pb.MoreRepeated{XXX_unrecognized: []byte{13<<3 | 0, 4}}}, {"extension (unencoded)", messageWithExtension1}, {"extension (encoded)", messageWithExtension3}, + // proto3 message + {"proto3 empty", &proto3pb.Message{}}, + {"proto3 bool", &proto3pb.Message{TrueScotsman: true}}, + {"proto3 int64", &proto3pb.Message{ResultCount: 1}}, + {"proto3 uint32", &proto3pb.Message{HeightInCm: 123}}, + {"proto3 float", &proto3pb.Message{Score: 12.6}}, + {"proto3 string", &proto3pb.Message{Name: "Snezana"}}, + {"proto3 bytes", &proto3pb.Message{Data: []byte("wowsa")}}, + {"proto3 bytes, empty", &proto3pb.Message{Data: []byte{}}}, + {"proto3 enum", &proto3pb.Message{Hilarity: proto3pb.Message_PUNS}}, + {"proto3 map field with empty bytes", &proto3pb.MessageWithMap{ByteMapping: map[bool][]byte{false: {}}}}, + + {"map field", &pb.MessageWithMap{NameMapping: map[int32]string{1: "Rob", 7: "Andrew"}}}, + {"map field with message", &pb.MessageWithMap{MsgMapping: map[int64]*pb.FloatingPoint{0x7001: {F: Float64(2.0)}}}}, + {"map field with bytes", &pb.MessageWithMap{ByteMapping: map[bool][]byte{true: []byte("this time for sure")}}}, + {"map field with empty bytes", &pb.MessageWithMap{ByteMapping: map[bool][]byte{true: {}}}}, + + {"map field with big entry", &pb.MessageWithMap{NameMapping: map[int32]string{8: strings.Repeat("x", 125)}}}, + {"map field with big key and val", &pb.MessageWithMap{StrToStr: map[string]string{strings.Repeat("x", 70): strings.Repeat("y", 70)}}}, + {"map field with big numeric key", &pb.MessageWithMap{NameMapping: map[int32]string{0xf00d: "om nom nom"}}}, } func TestSize(t *testing.T) { diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/skip_gogo.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/skip_gogo.go index d8683c8ed..4fe7e0815 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/skip_gogo.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/skip_gogo.go @@ -80,7 +80,7 @@ func Skip(data []byte) (n int, err error) { return index, nil case 3: for { - var wire uint64 + var innerWire uint64 var start int = index for shift := uint(0); ; shift += 7 { if index >= l { @@ -88,13 +88,13 @@ func Skip(data []byte) (n int, err error) { } b := data[index] index++ - wire |= (uint64(b) & 0x7F) << shift + innerWire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } - wireType := int(wire & 0x7) - if wireType == 4 { + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { break } next, err := Skip(data[start:]) diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata/Makefile b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata/Makefile index beb438e3e..1e676c37f 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata/Makefile +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata/Makefile @@ -32,16 +32,6 @@ all: regenerate regenerate: - rm -f test.pb.go - protoc --gogo_out=. test.proto + go install github.com/gogo/protobuf/protoc-gen-gogo/version/protoc-min-version + protoc-min-version --version="3.0.0" --gogo_out=. test.proto -# The following rules are just aids to development. Not needed for typical testing. - -diff: regenerate - hg diff test.pb.go - -restore: - cp test.pb.go.golden test.pb.go - -preserve: - cp test.pb.go test.pb.go.golden diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata/test.pb.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata/test.pb.go index 77e4821b2..6d6e97cda 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata/test.pb.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata/test.pb.go @@ -33,6 +33,7 @@ It has these top-level messages: GroupOld GroupNew FloatingPoint + MessageWithMap */ package testdata @@ -1885,6 +1886,46 @@ func (m *FloatingPoint) GetF() float64 { return 0 } +type MessageWithMap struct { + NameMapping map[int32]string `protobuf:"bytes,1,rep,name=name_mapping" json:"name_mapping,omitempty" protobuf_key:"varint,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + MsgMapping map[int64]*FloatingPoint `protobuf:"bytes,2,rep,name=msg_mapping" json:"msg_mapping,omitempty" protobuf_key:"zigzag64,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + ByteMapping map[bool][]byte `protobuf:"bytes,3,rep,name=byte_mapping" json:"byte_mapping,omitempty" protobuf_key:"varint,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + StrToStr map[string]string `protobuf:"bytes,4,rep,name=str_to_str" json:"str_to_str,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *MessageWithMap) Reset() { *m = MessageWithMap{} } +func (m *MessageWithMap) String() string { return proto.CompactTextString(m) } +func (*MessageWithMap) ProtoMessage() {} + +func (m *MessageWithMap) GetNameMapping() map[int32]string { + if m != nil { + return m.NameMapping + } + return nil +} + +func (m *MessageWithMap) GetMsgMapping() map[int64]*FloatingPoint { + if m != nil { + return m.MsgMapping + } + return nil +} + +func (m *MessageWithMap) GetByteMapping() map[bool][]byte { + if m != nil { + return m.ByteMapping + } + return nil +} + +func (m *MessageWithMap) GetStrToStr() map[string]string { + if m != nil { + return m.StrToStr + } + return nil +} + var E_Greeting = &proto.ExtensionDesc{ ExtendedType: (*MyMessage)(nil), ExtensionType: ([]string)(nil), diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata/test.proto b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata/test.proto index ac9542abc..411634d82 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata/test.proto +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata/test.proto @@ -426,3 +426,10 @@ message GroupNew { message FloatingPoint { required double f = 1; } + +message MessageWithMap { + map name_mapping = 1; + map msg_mapping = 2; + map byte_mapping = 3; + map str_to_str = 4; +} diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text.go index af6178989..5d77a9420 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text.go @@ -253,6 +253,100 @@ func writeStruct(w *textWriter, sv reflect.Value) error { } continue } + if fv.Kind() == reflect.Map { + // Map fields are rendered as a repeated struct with key/value fields. + keys := fv.MapKeys() // TODO: should we sort these for deterministic output? + sort.Sort(mapKeys(keys)) + for _, key := range keys { + val := fv.MapIndex(key) + if err := writeName(w, props); err != nil { + return err + } + if !w.compact { + if err := w.WriteByte(' '); err != nil { + return err + } + } + // open struct + if err := w.WriteByte('<'); err != nil { + return err + } + if !w.compact { + if err := w.WriteByte('\n'); err != nil { + return err + } + } + w.indent() + // key + if _, err := w.WriteString("key:"); err != nil { + return err + } + if !w.compact { + if err := w.WriteByte(' '); err != nil { + return err + } + } + if err := writeAny(w, key, props.mkeyprop); err != nil { + return err + } + if err := w.WriteByte('\n'); err != nil { + return err + } + // value + if _, err := w.WriteString("value:"); err != nil { + return err + } + if !w.compact { + if err := w.WriteByte(' '); err != nil { + return err + } + } + if err := writeAny(w, val, props.mvalprop); err != nil { + return err + } + if err := w.WriteByte('\n'); err != nil { + return err + } + // close struct + w.unindent() + if err := w.WriteByte('>'); err != nil { + return err + } + if err := w.WriteByte('\n'); err != nil { + return err + } + } + continue + } + if props.proto3 && fv.Kind() == reflect.Slice && fv.Len() == 0 { + // empty bytes field + continue + } + if props.proto3 && fv.Kind() != reflect.Ptr && fv.Kind() != reflect.Slice { + // proto3 non-repeated scalar field; skip if zero value + switch fv.Kind() { + case reflect.Bool: + if !fv.Bool() { + continue + } + case reflect.Int32, reflect.Int64: + if fv.Int() == 0 { + continue + } + case reflect.Uint32, reflect.Uint64: + if fv.Uint() == 0 { + continue + } + case reflect.Float32, reflect.Float64: + if fv.Float() == 0 { + continue + } + case reflect.String: + if fv.String() == "" { + continue + } + } + } if err := writeName(w, props); err != nil { return err @@ -354,7 +448,7 @@ func writeAny(w *textWriter, v reflect.Value, props *Properties) error { switch v.Kind() { case reflect.Slice: // Should only be a []byte; repeated fields are handled in writeStruct. - if err := writeString(w, string(v.Interface().([]byte))); err != nil { + if err := writeString(w, string(v.Bytes())); err != nil { return err } case reflect.String: diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text_parser.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text_parser.go index 151bf73ea..f769937bd 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text_parser.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text_parser.go @@ -360,6 +360,18 @@ func (p *textParser) next() *token { return &p.cur } +func (p *textParser) consumeToken(s string) error { + tok := p.next() + if tok.err != nil { + return tok.err + } + if tok.value != s { + p.back() + return p.errorf("expected %q, found %q", s, tok.value) + } + return nil +} + // Return a RequiredNotSetError indicating which required field was not set. func (p *textParser) missingRequiredFieldError(sv reflect.Value) *RequiredNotSetError { st := sv.Type() @@ -414,6 +426,10 @@ func (p *textParser) checkForColon(props *Properties, typ reflect.Type) *ParseEr if typ.Elem().Kind() != reflect.Ptr { break } + } else if typ.Kind() == reflect.String { + // The proto3 exception is for a string field, + // which requires a colon. + break } needColon = false } @@ -519,6 +535,60 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { dst := sv.Field(fi) + if dst.Kind() == reflect.Map { + // Consume any colon. + if err := p.checkForColon(props, dst.Type()); err != nil { + return err + } + + // Construct the map if it doesn't already exist. + if dst.IsNil() { + dst.Set(reflect.MakeMap(dst.Type())) + } + key := reflect.New(dst.Type().Key()).Elem() + val := reflect.New(dst.Type().Elem()).Elem() + + // The map entry should be this sequence of tokens: + // < key : KEY value : VALUE > + // Technically the "key" and "value" could come in any order, + // but in practice they won't. + + tok := p.next() + var terminator string + switch tok.value { + case "<": + terminator = ">" + case "{": + terminator = "}" + default: + return p.errorf("expected '{' or '<', found %q", tok.value) + } + if err := p.consumeToken("key"); err != nil { + return err + } + if err := p.consumeToken(":"); err != nil { + return err + } + if err := p.readAny(key, props.mkeyprop); err != nil { + return err + } + if err := p.consumeToken("value"); err != nil { + return err + } + if err := p.checkForColon(props.mvalprop, dst.Type().Elem()); err != nil { + return err + } + if err := p.readAny(val, props.mvalprop); err != nil { + return err + } + if err := p.consumeToken(terminator); err != nil { + return err + } + + dst.SetMapIndex(key, val) + continue + } + // Check that it's not already set if it's not a repeated field. if !props.Repeated && fieldSet[name] { return p.errorf("non-repeated field %q was repeated", name) diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text_parser_test.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text_parser_test.go index 248576792..ce1a9e8df 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text_parser_test.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text_parser_test.go @@ -36,8 +36,9 @@ import ( "reflect" "testing" - . "./testdata" . "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto" + proto3pb "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_proto" + . "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata" ) type UnmarshalTextTest struct { @@ -443,6 +444,48 @@ func TestRepeatedEnum(t *testing.T) { } } +func TestProto3TextParsing(t *testing.T) { + m := new(proto3pb.Message) + const in = `name: "Wallace" true_scotsman: true` + want := &proto3pb.Message{ + Name: "Wallace", + TrueScotsman: true, + } + if err := UnmarshalText(in, m); err != nil { + t.Fatal(err) + } + if !Equal(m, want) { + t.Errorf("\n got %v\nwant %v", m, want) + } +} + +func TestMapParsing(t *testing.T) { + m := new(MessageWithMap) + const in = `name_mapping: name_mapping:` + + `msg_mapping:>` + + `msg_mapping>` + // no colon after "value" + `byte_mapping:` + want := &MessageWithMap{ + NameMapping: map[int32]string{ + 1: "Beatles", + 1234: "Feist", + }, + MsgMapping: map[int64]*FloatingPoint{ + -4: {F: Float64(2.0)}, + -2: {F: Float64(4.0)}, + }, + ByteMapping: map[bool][]byte{ + true: []byte("so be it"), + }, + } + if err := UnmarshalText(in, m); err != nil { + t.Fatal(err) + } + if !Equal(m, want) { + t.Errorf("\n got %v\nwant %v", m, want) + } +} + var benchInput string func init() { diff --git a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text_test.go b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text_test.go index 09b1fd9b2..dd462c22b 100644 --- a/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text_test.go +++ b/Godeps/_workspace/src/github.com/gogo/protobuf/proto/text_test.go @@ -40,8 +40,8 @@ import ( "testing" "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto" - - pb "./testdata" + proto3pb "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/proto3_proto" + pb "github.com/coreos/etcd/Godeps/_workspace/src/github.com/gogo/protobuf/proto/testdata" ) // textMessage implements the methods that allow it to marshal and unmarshal @@ -406,3 +406,30 @@ Message t.Errorf(" got: %s\nwant: %s", s, want) } } + +func TestProto3Text(t *testing.T) { + tests := []struct { + m proto.Message + want string + }{ + // zero message + {&proto3pb.Message{}, ``}, + // zero message except for an empty byte slice + {&proto3pb.Message{Data: []byte{}}, ``}, + // trivial case + {&proto3pb.Message{Name: "Rob", HeightInCm: 175}, `name:"Rob" height_in_cm:175`}, + // empty map + {&pb.MessageWithMap{}, ``}, + // non-empty map; current map format is the same as a repeated struct + { + &pb.MessageWithMap{NameMapping: map[int32]string{1234: "Feist"}}, + `name_mapping:`, + }, + } + for _, test := range tests { + got := strings.TrimSpace(test.m.String()) + if got != test.want { + t.Errorf("\n got %s\nwant %s", got, test.want) + } + } +}