256
|
1 |
// Copyright 2019 The Go Authors. All rights reserved. |
|
2 |
// Use of this source code is governed by a BSD-style |
|
3 |
// license that can be found in the LICENSE file. |
|
4 |
|
|
5 |
package proto |
|
6 |
|
|
7 |
import ( |
|
8 |
"google.golang.org/protobuf/encoding/protowire" |
|
9 |
"google.golang.org/protobuf/internal/encoding/messageset" |
|
10 |
"google.golang.org/protobuf/internal/order" |
|
11 |
"google.golang.org/protobuf/internal/pragma" |
|
12 |
"google.golang.org/protobuf/reflect/protoreflect" |
|
13 |
"google.golang.org/protobuf/runtime/protoiface" |
|
14 |
) |
|
15 |
|
|
16 |
// MarshalOptions configures the marshaler. |
|
17 |
// |
|
18 |
// Example usage: |
260
|
19 |
// |
|
20 |
// b, err := MarshalOptions{Deterministic: true}.Marshal(m) |
256
|
21 |
type MarshalOptions struct { |
|
22 |
pragma.NoUnkeyedLiterals |
|
23 |
|
|
24 |
// AllowPartial allows messages that have missing required fields to marshal |
|
25 |
// without returning an error. If AllowPartial is false (the default), |
|
26 |
// Marshal will return an error if there are any missing required fields. |
|
27 |
AllowPartial bool |
|
28 |
|
|
29 |
// Deterministic controls whether the same message will always be |
|
30 |
// serialized to the same bytes within the same binary. |
|
31 |
// |
|
32 |
// Setting this option guarantees that repeated serialization of |
|
33 |
// the same message will return the same bytes, and that different |
|
34 |
// processes of the same binary (which may be executing on different |
|
35 |
// machines) will serialize equal messages to the same bytes. |
|
36 |
// It has no effect on the resulting size of the encoded message compared |
|
37 |
// to a non-deterministic marshal. |
|
38 |
// |
|
39 |
// Note that the deterministic serialization is NOT canonical across |
|
40 |
// languages. It is not guaranteed to remain stable over time. It is |
|
41 |
// unstable across different builds with schema changes due to unknown |
|
42 |
// fields. Users who need canonical serialization (e.g., persistent |
|
43 |
// storage in a canonical form, fingerprinting, etc.) must define |
|
44 |
// their own canonicalization specification and implement their own |
|
45 |
// serializer rather than relying on this API. |
|
46 |
// |
|
47 |
// If deterministic serialization is requested, map entries will be |
|
48 |
// sorted by keys in lexographical order. This is an implementation |
|
49 |
// detail and subject to change. |
|
50 |
Deterministic bool |
|
51 |
|
|
52 |
// UseCachedSize indicates that the result of a previous Size call |
|
53 |
// may be reused. |
|
54 |
// |
|
55 |
// Setting this option asserts that: |
|
56 |
// |
|
57 |
// 1. Size has previously been called on this message with identical |
|
58 |
// options (except for UseCachedSize itself). |
|
59 |
// |
|
60 |
// 2. The message and all its submessages have not changed in any |
|
61 |
// way since the Size call. |
|
62 |
// |
|
63 |
// If either of these invariants is violated, |
|
64 |
// the results are undefined and may include panics or corrupted output. |
|
65 |
// |
|
66 |
// Implementations MAY take this option into account to provide |
|
67 |
// better performance, but there is no guarantee that they will do so. |
|
68 |
// There is absolutely no guarantee that Size followed by Marshal with |
|
69 |
// UseCachedSize set will perform equivalently to Marshal alone. |
|
70 |
UseCachedSize bool |
|
71 |
} |
|
72 |
|
|
73 |
// Marshal returns the wire-format encoding of m. |
|
74 |
func Marshal(m Message) ([]byte, error) { |
|
75 |
// Treat nil message interface as an empty message; nothing to output. |
|
76 |
if m == nil { |
|
77 |
return nil, nil |
|
78 |
} |
|
79 |
|
|
80 |
out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect()) |
|
81 |
if len(out.Buf) == 0 && err == nil { |
|
82 |
out.Buf = emptyBytesForMessage(m) |
|
83 |
} |
|
84 |
return out.Buf, err |
|
85 |
} |
|
86 |
|
|
87 |
// Marshal returns the wire-format encoding of m. |
|
88 |
func (o MarshalOptions) Marshal(m Message) ([]byte, error) { |
|
89 |
// Treat nil message interface as an empty message; nothing to output. |
|
90 |
if m == nil { |
|
91 |
return nil, nil |
|
92 |
} |
|
93 |
|
|
94 |
out, err := o.marshal(nil, m.ProtoReflect()) |
|
95 |
if len(out.Buf) == 0 && err == nil { |
|
96 |
out.Buf = emptyBytesForMessage(m) |
|
97 |
} |
|
98 |
return out.Buf, err |
|
99 |
} |
|
100 |
|
|
101 |
// emptyBytesForMessage returns a nil buffer if and only if m is invalid, |
|
102 |
// otherwise it returns a non-nil empty buffer. |
|
103 |
// |
|
104 |
// This is to assist the edge-case where user-code does the following: |
260
|
105 |
// |
256
|
106 |
// m1.OptionalBytes, _ = proto.Marshal(m2) |
260
|
107 |
// |
256
|
108 |
// where they expect the proto2 "optional_bytes" field to be populated |
|
109 |
// if any only if m2 is a valid message. |
|
110 |
func emptyBytesForMessage(m Message) []byte { |
|
111 |
if m == nil || !m.ProtoReflect().IsValid() { |
|
112 |
return nil |
|
113 |
} |
|
114 |
return emptyBuf[:] |
|
115 |
} |
|
116 |
|
|
117 |
// MarshalAppend appends the wire-format encoding of m to b, |
|
118 |
// returning the result. |
|
119 |
func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) { |
|
120 |
// Treat nil message interface as an empty message; nothing to append. |
|
121 |
if m == nil { |
|
122 |
return b, nil |
|
123 |
} |
|
124 |
|
|
125 |
out, err := o.marshal(b, m.ProtoReflect()) |
|
126 |
return out.Buf, err |
|
127 |
} |
|
128 |
|
|
129 |
// MarshalState returns the wire-format encoding of a message. |
|
130 |
// |
|
131 |
// This method permits fine-grained control over the marshaler. |
|
132 |
// Most users should use Marshal instead. |
|
133 |
func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) { |
|
134 |
return o.marshal(in.Buf, in.Message) |
|
135 |
} |
|
136 |
|
|
137 |
// marshal is a centralized function that all marshal operations go through. |
|
138 |
// For profiling purposes, avoid changing the name of this function or |
|
139 |
// introducing other code paths for marshal that do not go through this. |
|
140 |
func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) { |
|
141 |
allowPartial := o.AllowPartial |
|
142 |
o.AllowPartial = true |
|
143 |
if methods := protoMethods(m); methods != nil && methods.Marshal != nil && |
|
144 |
!(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) { |
|
145 |
in := protoiface.MarshalInput{ |
|
146 |
Message: m, |
|
147 |
Buf: b, |
|
148 |
} |
|
149 |
if o.Deterministic { |
|
150 |
in.Flags |= protoiface.MarshalDeterministic |
|
151 |
} |
|
152 |
if o.UseCachedSize { |
|
153 |
in.Flags |= protoiface.MarshalUseCachedSize |
|
154 |
} |
|
155 |
if methods.Size != nil { |
|
156 |
sout := methods.Size(protoiface.SizeInput{ |
|
157 |
Message: m, |
|
158 |
Flags: in.Flags, |
|
159 |
}) |
|
160 |
if cap(b) < len(b)+sout.Size { |
|
161 |
in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size)) |
|
162 |
copy(in.Buf, b) |
|
163 |
} |
|
164 |
in.Flags |= protoiface.MarshalUseCachedSize |
|
165 |
} |
|
166 |
out, err = methods.Marshal(in) |
|
167 |
} else { |
|
168 |
out.Buf, err = o.marshalMessageSlow(b, m) |
|
169 |
} |
|
170 |
if err != nil { |
|
171 |
return out, err |
|
172 |
} |
|
173 |
if allowPartial { |
|
174 |
return out, nil |
|
175 |
} |
|
176 |
return out, checkInitialized(m) |
|
177 |
} |
|
178 |
|
|
179 |
func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) { |
|
180 |
out, err := o.marshal(b, m) |
|
181 |
return out.Buf, err |
|
182 |
} |
|
183 |
|
|
184 |
// growcap scales up the capacity of a slice. |
|
185 |
// |
|
186 |
// Given a slice with a current capacity of oldcap and a desired |
|
187 |
// capacity of wantcap, growcap returns a new capacity >= wantcap. |
|
188 |
// |
|
189 |
// The algorithm is mostly identical to the one used by append as of Go 1.14. |
|
190 |
func growcap(oldcap, wantcap int) (newcap int) { |
|
191 |
if wantcap > oldcap*2 { |
|
192 |
newcap = wantcap |
|
193 |
} else if oldcap < 1024 { |
|
194 |
// The Go 1.14 runtime takes this case when len(s) < 1024, |
|
195 |
// not when cap(s) < 1024. The difference doesn't seem |
|
196 |
// significant here. |
|
197 |
newcap = oldcap * 2 |
|
198 |
} else { |
|
199 |
newcap = oldcap |
|
200 |
for 0 < newcap && newcap < wantcap { |
|
201 |
newcap += newcap / 4 |
|
202 |
} |
|
203 |
if newcap <= 0 { |
|
204 |
newcap = wantcap |
|
205 |
} |
|
206 |
} |
|
207 |
return newcap |
|
208 |
} |
|
209 |
|
|
210 |
func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) { |
|
211 |
if messageset.IsMessageSet(m.Descriptor()) { |
|
212 |
return o.marshalMessageSet(b, m) |
|
213 |
} |
|
214 |
fieldOrder := order.AnyFieldOrder |
|
215 |
if o.Deterministic { |
|
216 |
// TODO: This should use a more natural ordering like NumberFieldOrder, |
|
217 |
// but doing so breaks golden tests that make invalid assumption about |
|
218 |
// output stability of this implementation. |
|
219 |
fieldOrder = order.LegacyFieldOrder |
|
220 |
} |
|
221 |
var err error |
|
222 |
order.RangeFields(m, fieldOrder, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { |
|
223 |
b, err = o.marshalField(b, fd, v) |
|
224 |
return err == nil |
|
225 |
}) |
|
226 |
if err != nil { |
|
227 |
return b, err |
|
228 |
} |
|
229 |
b = append(b, m.GetUnknown()...) |
|
230 |
return b, nil |
|
231 |
} |
|
232 |
|
|
233 |
func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) { |
|
234 |
switch { |
|
235 |
case fd.IsList(): |
|
236 |
return o.marshalList(b, fd, value.List()) |
|
237 |
case fd.IsMap(): |
|
238 |
return o.marshalMap(b, fd, value.Map()) |
|
239 |
default: |
|
240 |
b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()]) |
|
241 |
return o.marshalSingular(b, fd, value) |
|
242 |
} |
|
243 |
} |
|
244 |
|
|
245 |
func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) { |
|
246 |
if fd.IsPacked() && list.Len() > 0 { |
|
247 |
b = protowire.AppendTag(b, fd.Number(), protowire.BytesType) |
|
248 |
b, pos := appendSpeculativeLength(b) |
|
249 |
for i, llen := 0, list.Len(); i < llen; i++ { |
|
250 |
var err error |
|
251 |
b, err = o.marshalSingular(b, fd, list.Get(i)) |
|
252 |
if err != nil { |
|
253 |
return b, err |
|
254 |
} |
|
255 |
} |
|
256 |
b = finishSpeculativeLength(b, pos) |
|
257 |
return b, nil |
|
258 |
} |
|
259 |
|
|
260 |
kind := fd.Kind() |
|
261 |
for i, llen := 0, list.Len(); i < llen; i++ { |
|
262 |
var err error |
|
263 |
b = protowire.AppendTag(b, fd.Number(), wireTypes[kind]) |
|
264 |
b, err = o.marshalSingular(b, fd, list.Get(i)) |
|
265 |
if err != nil { |
|
266 |
return b, err |
|
267 |
} |
|
268 |
} |
|
269 |
return b, nil |
|
270 |
} |
|
271 |
|
|
272 |
func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) { |
|
273 |
keyf := fd.MapKey() |
|
274 |
valf := fd.MapValue() |
|
275 |
keyOrder := order.AnyKeyOrder |
|
276 |
if o.Deterministic { |
|
277 |
keyOrder = order.GenericKeyOrder |
|
278 |
} |
|
279 |
var err error |
|
280 |
order.RangeEntries(mapv, keyOrder, func(key protoreflect.MapKey, value protoreflect.Value) bool { |
|
281 |
b = protowire.AppendTag(b, fd.Number(), protowire.BytesType) |
|
282 |
var pos int |
|
283 |
b, pos = appendSpeculativeLength(b) |
|
284 |
|
|
285 |
b, err = o.marshalField(b, keyf, key.Value()) |
|
286 |
if err != nil { |
|
287 |
return false |
|
288 |
} |
|
289 |
b, err = o.marshalField(b, valf, value) |
|
290 |
if err != nil { |
|
291 |
return false |
|
292 |
} |
|
293 |
b = finishSpeculativeLength(b, pos) |
|
294 |
return true |
|
295 |
}) |
|
296 |
return b, err |
|
297 |
} |
|
298 |
|
|
299 |
// When encoding length-prefixed fields, we speculatively set aside some number of bytes |
|
300 |
// for the length, encode the data, and then encode the length (shifting the data if necessary |
|
301 |
// to make room). |
|
302 |
const speculativeLength = 1 |
|
303 |
|
|
304 |
func appendSpeculativeLength(b []byte) ([]byte, int) { |
|
305 |
pos := len(b) |
|
306 |
b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...) |
|
307 |
return b, pos |
|
308 |
} |
|
309 |
|
|
310 |
func finishSpeculativeLength(b []byte, pos int) []byte { |
|
311 |
mlen := len(b) - pos - speculativeLength |
|
312 |
msiz := protowire.SizeVarint(uint64(mlen)) |
|
313 |
if msiz != speculativeLength { |
|
314 |
for i := 0; i < msiz-speculativeLength; i++ { |
|
315 |
b = append(b, 0) |
|
316 |
} |
|
317 |
copy(b[pos+msiz:], b[pos+speculativeLength:]) |
|
318 |
b = b[:pos+msiz+mlen] |
|
319 |
} |
|
320 |
protowire.AppendVarint(b[:pos], uint64(mlen)) |
|
321 |
return b |
|
322 |
} |