1 // Copyright 2019 The Go Authors. All rights reserved.

     2 // Use of this source code is governed by a BSD-style

     3 // license that can be found in the LICENSE file.

     4 

     5 package proto

     6 

     7 import (

     8 	"google.golang.org/protobuf/encoding/protowire"

     9 	"google.golang.org/protobuf/internal/encoding/messageset"

    10 	"google.golang.org/protobuf/internal/order"

    11 	"google.golang.org/protobuf/internal/pragma"

    12 	"google.golang.org/protobuf/reflect/protoreflect"

    13 	"google.golang.org/protobuf/runtime/protoiface"

    14 )

    15 

    16 // MarshalOptions configures the marshaler.

    17 //

    18 // Example usage:

    19 //   b, err := MarshalOptions{Deterministic: true}.Marshal(m)

    20 type MarshalOptions struct {

    21 	pragma.NoUnkeyedLiterals

    22 

    23 	// AllowPartial allows messages that have missing required fields to marshal

    24 	// without returning an error. If AllowPartial is false (the default),

    25 	// Marshal will return an error if there are any missing required fields.

    26 	AllowPartial bool

    27 

    28 	// Deterministic controls whether the same message will always be

    29 	// serialized to the same bytes within the same binary.

    30 	//

    31 	// Setting this option guarantees that repeated serialization of

    32 	// the same message will return the same bytes, and that different

    33 	// processes of the same binary (which may be executing on different

    34 	// machines) will serialize equal messages to the same bytes.

    35 	// It has no effect on the resulting size of the encoded message compared

    36 	// to a non-deterministic marshal.

    37 	//

    38 	// Note that the deterministic serialization is NOT canonical across

    39 	// languages. It is not guaranteed to remain stable over time. It is

    40 	// unstable across different builds with schema changes due to unknown

    41 	// fields. Users who need canonical serialization (e.g., persistent

    42 	// storage in a canonical form, fingerprinting, etc.) must define

    43 	// their own canonicalization specification and implement their own

    44 	// serializer rather than relying on this API.

    45 	//

    46 	// If deterministic serialization is requested, map entries will be

    47 	// sorted by keys in lexographical order. This is an implementation

    48 	// detail and subject to change.

    49 	Deterministic bool

    50 

    51 	// UseCachedSize indicates that the result of a previous Size call

    52 	// may be reused.

    53 	//

    54 	// Setting this option asserts that:

    55 	//

    56 	// 1. Size has previously been called on this message with identical

    57 	// options (except for UseCachedSize itself).

    58 	//

    59 	// 2. The message and all its submessages have not changed in any

    60 	// way since the Size call.

    61 	//

    62 	// If either of these invariants is violated,

    63 	// the results are undefined and may include panics or corrupted output.

    64 	//

    65 	// Implementations MAY take this option into account to provide

    66 	// better performance, but there is no guarantee that they will do so.

    67 	// There is absolutely no guarantee that Size followed by Marshal with

    68 	// UseCachedSize set will perform equivalently to Marshal alone.

    69 	UseCachedSize bool

    70 }

    71 

    72 // Marshal returns the wire-format encoding of m.

    73 func Marshal(m Message) ([]byte, error) {

    74 	// Treat nil message interface as an empty message; nothing to output.

    75 	if m == nil {

    76 		return nil, nil

    77 	}

    78 

    79 	out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect())

    80 	if len(out.Buf) == 0 && err == nil {

    81 		out.Buf = emptyBytesForMessage(m)

    82 	}

    83 	return out.Buf, err

    84 }

    85 

    86 // Marshal returns the wire-format encoding of m.

    87 func (o MarshalOptions) Marshal(m Message) ([]byte, error) {

    88 	// Treat nil message interface as an empty message; nothing to output.

    89 	if m == nil {

    90 		return nil, nil

    91 	}

    92 

    93 	out, err := o.marshal(nil, m.ProtoReflect())

    94 	if len(out.Buf) == 0 && err == nil {

    95 		out.Buf = emptyBytesForMessage(m)

    96 	}

    97 	return out.Buf, err

    98 }

    99 

   100 // emptyBytesForMessage returns a nil buffer if and only if m is invalid,

   101 // otherwise it returns a non-nil empty buffer.

   102 //

   103 // This is to assist the edge-case where user-code does the following:

   104 //	m1.OptionalBytes, _ = proto.Marshal(m2)

   105 // where they expect the proto2 "optional_bytes" field to be populated

   106 // if any only if m2 is a valid message.

   107 func emptyBytesForMessage(m Message) []byte {

   108 	if m == nil || !m.ProtoReflect().IsValid() {

   109 		return nil

   110 	}

   111 	return emptyBuf[:]

   112 }

   113 

   114 // MarshalAppend appends the wire-format encoding of m to b,

   115 // returning the result.

   116 func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) {

   117 	// Treat nil message interface as an empty message; nothing to append.

   118 	if m == nil {

   119 		return b, nil

   120 	}

   121 

   122 	out, err := o.marshal(b, m.ProtoReflect())

   123 	return out.Buf, err

   124 }

   125 

   126 // MarshalState returns the wire-format encoding of a message.

   127 //

   128 // This method permits fine-grained control over the marshaler.

   129 // Most users should use Marshal instead.

   130 func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) {

   131 	return o.marshal(in.Buf, in.Message)

   132 }

   133 

   134 // marshal is a centralized function that all marshal operations go through.

   135 // For profiling purposes, avoid changing the name of this function or

   136 // introducing other code paths for marshal that do not go through this.

   137 func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) {

   138 	allowPartial := o.AllowPartial

   139 	o.AllowPartial = true

   140 	if methods := protoMethods(m); methods != nil && methods.Marshal != nil &&

   141 		!(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) {

   142 		in := protoiface.MarshalInput{

   143 			Message: m,

   144 			Buf:     b,

   145 		}

   146 		if o.Deterministic {

   147 			in.Flags |= protoiface.MarshalDeterministic

   148 		}

   149 		if o.UseCachedSize {

   150 			in.Flags |= protoiface.MarshalUseCachedSize

   151 		}

   152 		if methods.Size != nil {

   153 			sout := methods.Size(protoiface.SizeInput{

   154 				Message: m,

   155 				Flags:   in.Flags,

   156 			})

   157 			if cap(b) < len(b)+sout.Size {

   158 				in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size))

   159 				copy(in.Buf, b)

   160 			}

   161 			in.Flags |= protoiface.MarshalUseCachedSize

   162 		}

   163 		out, err = methods.Marshal(in)

   164 	} else {

   165 		out.Buf, err = o.marshalMessageSlow(b, m)

   166 	}

   167 	if err != nil {

   168 		return out, err

   169 	}

   170 	if allowPartial {

   171 		return out, nil

   172 	}

   173 	return out, checkInitialized(m)

   174 }

   175 

   176 func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) {

   177 	out, err := o.marshal(b, m)

   178 	return out.Buf, err

   179 }

   180 

   181 // growcap scales up the capacity of a slice.

   182 //

   183 // Given a slice with a current capacity of oldcap and a desired

   184 // capacity of wantcap, growcap returns a new capacity >= wantcap.

   185 //

   186 // The algorithm is mostly identical to the one used by append as of Go 1.14.

   187 func growcap(oldcap, wantcap int) (newcap int) {

   188 	if wantcap > oldcap*2 {

   189 		newcap = wantcap

   190 	} else if oldcap < 1024 {

   191 		// The Go 1.14 runtime takes this case when len(s) < 1024,

   192 		// not when cap(s) < 1024. The difference doesn't seem

   193 		// significant here.

   194 		newcap = oldcap * 2

   195 	} else {

   196 		newcap = oldcap

   197 		for 0 < newcap && newcap < wantcap {

   198 			newcap += newcap / 4

   199 		}

   200 		if newcap <= 0 {

   201 			newcap = wantcap

   202 		}

   203 	}

   204 	return newcap

   205 }

   206 

   207 func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) {

   208 	if messageset.IsMessageSet(m.Descriptor()) {

   209 		return o.marshalMessageSet(b, m)

   210 	}

   211 	fieldOrder := order.AnyFieldOrder

   212 	if o.Deterministic {

   213 		// TODO: This should use a more natural ordering like NumberFieldOrder,

   214 		// but doing so breaks golden tests that make invalid assumption about

   215 		// output stability of this implementation.

   216 		fieldOrder = order.LegacyFieldOrder

   217 	}

   218 	var err error

   219 	order.RangeFields(m, fieldOrder, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {

   220 		b, err = o.marshalField(b, fd, v)

   221 		return err == nil

   222 	})

   223 	if err != nil {

   224 		return b, err

   225 	}

   226 	b = append(b, m.GetUnknown()...)

   227 	return b, nil

   228 }

   229 

   230 func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) {

   231 	switch {

   232 	case fd.IsList():

   233 		return o.marshalList(b, fd, value.List())

   234 	case fd.IsMap():

   235 		return o.marshalMap(b, fd, value.Map())

   236 	default:

   237 		b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()])

   238 		return o.marshalSingular(b, fd, value)

   239 	}

   240 }

   241 

   242 func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) {

   243 	if fd.IsPacked() && list.Len() > 0 {

   244 		b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)

   245 		b, pos := appendSpeculativeLength(b)

   246 		for i, llen := 0, list.Len(); i < llen; i++ {

   247 			var err error

   248 			b, err = o.marshalSingular(b, fd, list.Get(i))

   249 			if err != nil {

   250 				return b, err

   251 			}

   252 		}

   253 		b = finishSpeculativeLength(b, pos)

   254 		return b, nil

   255 	}

   256 

   257 	kind := fd.Kind()

   258 	for i, llen := 0, list.Len(); i < llen; i++ {

   259 		var err error

   260 		b = protowire.AppendTag(b, fd.Number(), wireTypes[kind])

   261 		b, err = o.marshalSingular(b, fd, list.Get(i))

   262 		if err != nil {

   263 			return b, err

   264 		}

   265 	}

   266 	return b, nil

   267 }

   268 

   269 func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) {

   270 	keyf := fd.MapKey()

   271 	valf := fd.MapValue()

   272 	keyOrder := order.AnyKeyOrder

   273 	if o.Deterministic {

   274 		keyOrder = order.GenericKeyOrder

   275 	}

   276 	var err error

   277 	order.RangeEntries(mapv, keyOrder, func(key protoreflect.MapKey, value protoreflect.Value) bool {

   278 		b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)

   279 		var pos int

   280 		b, pos = appendSpeculativeLength(b)

   281 

   282 		b, err = o.marshalField(b, keyf, key.Value())

   283 		if err != nil {

   284 			return false

   285 		}

   286 		b, err = o.marshalField(b, valf, value)

   287 		if err != nil {

   288 			return false

   289 		}

   290 		b = finishSpeculativeLength(b, pos)

   291 		return true

   292 	})

   293 	return b, err

   294 }

   295 

   296 // When encoding length-prefixed fields, we speculatively set aside some number of bytes

   297 // for the length, encode the data, and then encode the length (shifting the data if necessary

   298 // to make room).

   299 const speculativeLength = 1

   300 

   301 func appendSpeculativeLength(b []byte) ([]byte, int) {

   302 	pos := len(b)

   303 	b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...)

   304 	return b, pos

   305 }

   306 

   307 func finishSpeculativeLength(b []byte, pos int) []byte {

   308 	mlen := len(b) - pos - speculativeLength

   309 	msiz := protowire.SizeVarint(uint64(mlen))

   310 	if msiz != speculativeLength {

   311 		for i := 0; i < msiz-speculativeLength; i++ {

   312 			b = append(b, 0)

   313 		}

   314 		copy(b[pos+msiz:], b[pos+speculativeLength:])

   315 		b = b[:pos+msiz+mlen]

   316 	}

   317 	protowire.AppendVarint(b[:pos], uint64(mlen))

   318 	return b

   319 }