author | Mikael Berthe <mikael@lilotux.net> |
Sun, 16 Feb 2020 18:54:01 +0100 | |
changeset 251 | 1c52a0eeb952 |
parent 242 | 2a9ec03fe5a1 |
permissions | -rw-r--r-- |
242
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
1 |
// Copyright 2011 The Go Authors. All rights reserved. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
2 |
// Use of this source code is governed by a BSD-style |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
3 |
// license that can be found in the LICENSE file. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
4 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
5 |
package norm |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
6 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
7 |
import "unicode/utf8" |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
8 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
9 |
const ( |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
10 |
maxNonStarters = 30 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
11 |
// The maximum number of characters needed for a buffer is |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
12 |
// maxNonStarters + 1 for the starter + 1 for the GCJ |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
13 |
maxBufferSize = maxNonStarters + 2 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
14 |
maxNFCExpansion = 3 // NFC(0x1D160) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
15 |
maxNFKCExpansion = 18 // NFKC(0xFDFA) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
16 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
17 |
maxByteBufferSize = utf8.UTFMax * maxBufferSize // 128 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
18 |
) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
19 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
20 |
// ssState is used for reporting the segment state after inserting a rune. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
21 |
// It is returned by streamSafe.next. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
22 |
type ssState int |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
23 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
24 |
const ( |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
25 |
// Indicates a rune was successfully added to the segment. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
26 |
ssSuccess ssState = iota |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
27 |
// Indicates a rune starts a new segment and should not be added. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
28 |
ssStarter |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
29 |
// Indicates a rune caused a segment overflow and a CGJ should be inserted. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
30 |
ssOverflow |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
31 |
) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
32 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
33 |
// streamSafe implements the policy of when a CGJ should be inserted. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
34 |
type streamSafe uint8 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
35 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
36 |
// first inserts the first rune of a segment. It is a faster version of next if |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
37 |
// it is known p represents the first rune in a segment. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
38 |
func (ss *streamSafe) first(p Properties) { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
39 |
*ss = streamSafe(p.nTrailingNonStarters()) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
40 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
41 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
42 |
// insert returns a ssState value to indicate whether a rune represented by p |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
43 |
// can be inserted. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
44 |
func (ss *streamSafe) next(p Properties) ssState { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
45 |
if *ss > maxNonStarters { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
46 |
panic("streamSafe was not reset") |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
47 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
48 |
n := p.nLeadingNonStarters() |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
49 |
if *ss += streamSafe(n); *ss > maxNonStarters { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
50 |
*ss = 0 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
51 |
return ssOverflow |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
52 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
53 |
// The Stream-Safe Text Processing prescribes that the counting can stop |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
54 |
// as soon as a starter is encountered. However, there are some starters, |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
55 |
// like Jamo V and T, that can combine with other runes, leaving their |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
56 |
// successive non-starters appended to the previous, possibly causing an |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
57 |
// overflow. We will therefore consider any rune with a non-zero nLead to |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
58 |
// be a non-starter. Note that it always hold that if nLead > 0 then |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
59 |
// nLead == nTrail. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
60 |
if n == 0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
61 |
*ss = streamSafe(p.nTrailingNonStarters()) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
62 |
return ssStarter |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
63 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
64 |
return ssSuccess |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
65 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
66 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
67 |
// backwards is used for checking for overflow and segment starts |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
68 |
// when traversing a string backwards. Users do not need to call first |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
69 |
// for the first rune. The state of the streamSafe retains the count of |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
70 |
// the non-starters loaded. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
71 |
func (ss *streamSafe) backwards(p Properties) ssState { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
72 |
if *ss > maxNonStarters { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
73 |
panic("streamSafe was not reset") |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
74 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
75 |
c := *ss + streamSafe(p.nTrailingNonStarters()) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
76 |
if c > maxNonStarters { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
77 |
return ssOverflow |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
78 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
79 |
*ss = c |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
80 |
if p.nLeadingNonStarters() == 0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
81 |
return ssStarter |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
82 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
83 |
return ssSuccess |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
84 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
85 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
86 |
func (ss streamSafe) isMax() bool { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
87 |
return ss == maxNonStarters |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
88 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
89 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
90 |
// GraphemeJoiner is inserted after maxNonStarters non-starter runes. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
91 |
const GraphemeJoiner = "\u034F" |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
92 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
93 |
// reorderBuffer is used to normalize a single segment. Characters inserted with |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
94 |
// insert are decomposed and reordered based on CCC. The compose method can |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
95 |
// be used to recombine characters. Note that the byte buffer does not hold |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
96 |
// the UTF-8 characters in order. Only the rune array is maintained in sorted |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
97 |
// order. flush writes the resulting segment to a byte array. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
98 |
type reorderBuffer struct { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
99 |
rune [maxBufferSize]Properties // Per character info. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
100 |
byte [maxByteBufferSize]byte // UTF-8 buffer. Referenced by runeInfo.pos. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
101 |
nbyte uint8 // Number or bytes. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
102 |
ss streamSafe // For limiting length of non-starter sequence. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
103 |
nrune int // Number of runeInfos. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
104 |
f formInfo |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
105 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
106 |
src input |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
107 |
nsrc int |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
108 |
tmpBytes input |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
109 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
110 |
out []byte |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
111 |
flushF func(*reorderBuffer) bool |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
112 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
113 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
114 |
func (rb *reorderBuffer) init(f Form, src []byte) { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
115 |
rb.f = *formTable[f] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
116 |
rb.src.setBytes(src) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
117 |
rb.nsrc = len(src) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
118 |
rb.ss = 0 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
119 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
120 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
121 |
func (rb *reorderBuffer) initString(f Form, src string) { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
122 |
rb.f = *formTable[f] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
123 |
rb.src.setString(src) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
124 |
rb.nsrc = len(src) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
125 |
rb.ss = 0 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
126 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
127 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
128 |
func (rb *reorderBuffer) setFlusher(out []byte, f func(*reorderBuffer) bool) { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
129 |
rb.out = out |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
130 |
rb.flushF = f |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
131 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
132 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
133 |
// reset discards all characters from the buffer. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
134 |
func (rb *reorderBuffer) reset() { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
135 |
rb.nrune = 0 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
136 |
rb.nbyte = 0 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
137 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
138 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
139 |
func (rb *reorderBuffer) doFlush() bool { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
140 |
if rb.f.composing { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
141 |
rb.compose() |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
142 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
143 |
res := rb.flushF(rb) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
144 |
rb.reset() |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
145 |
return res |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
146 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
147 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
148 |
// appendFlush appends the normalized segment to rb.out. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
149 |
func appendFlush(rb *reorderBuffer) bool { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
150 |
for i := 0; i < rb.nrune; i++ { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
151 |
start := rb.rune[i].pos |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
152 |
end := start + rb.rune[i].size |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
153 |
rb.out = append(rb.out, rb.byte[start:end]...) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
154 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
155 |
return true |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
156 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
157 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
158 |
// flush appends the normalized segment to out and resets rb. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
159 |
func (rb *reorderBuffer) flush(out []byte) []byte { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
160 |
for i := 0; i < rb.nrune; i++ { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
161 |
start := rb.rune[i].pos |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
162 |
end := start + rb.rune[i].size |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
163 |
out = append(out, rb.byte[start:end]...) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
164 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
165 |
rb.reset() |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
166 |
return out |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
167 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
168 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
169 |
// flushCopy copies the normalized segment to buf and resets rb. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
170 |
// It returns the number of bytes written to buf. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
171 |
func (rb *reorderBuffer) flushCopy(buf []byte) int { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
172 |
p := 0 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
173 |
for i := 0; i < rb.nrune; i++ { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
174 |
runep := rb.rune[i] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
175 |
p += copy(buf[p:], rb.byte[runep.pos:runep.pos+runep.size]) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
176 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
177 |
rb.reset() |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
178 |
return p |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
179 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
180 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
181 |
// insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
182 |
// It returns false if the buffer is not large enough to hold the rune. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
183 |
// It is used internally by insert and insertString only. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
184 |
func (rb *reorderBuffer) insertOrdered(info Properties) { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
185 |
n := rb.nrune |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
186 |
b := rb.rune[:] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
187 |
cc := info.ccc |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
188 |
if cc > 0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
189 |
// Find insertion position + move elements to make room. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
190 |
for ; n > 0; n-- { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
191 |
if b[n-1].ccc <= cc { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
192 |
break |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
193 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
194 |
b[n] = b[n-1] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
195 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
196 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
197 |
rb.nrune += 1 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
198 |
pos := uint8(rb.nbyte) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
199 |
rb.nbyte += utf8.UTFMax |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
200 |
info.pos = pos |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
201 |
b[n] = info |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
202 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
203 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
204 |
// insertErr is an error code returned by insert. Using this type instead |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
205 |
// of error improves performance up to 20% for many of the benchmarks. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
206 |
type insertErr int |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
207 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
208 |
const ( |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
209 |
iSuccess insertErr = -iota |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
210 |
iShortDst |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
211 |
iShortSrc |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
212 |
) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
213 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
214 |
// insertFlush inserts the given rune in the buffer ordered by CCC. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
215 |
// If a decomposition with multiple segments are encountered, they leading |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
216 |
// ones are flushed. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
217 |
// It returns a non-zero error code if the rune was not inserted. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
218 |
func (rb *reorderBuffer) insertFlush(src input, i int, info Properties) insertErr { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
219 |
if rune := src.hangul(i); rune != 0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
220 |
rb.decomposeHangul(rune) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
221 |
return iSuccess |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
222 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
223 |
if info.hasDecomposition() { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
224 |
return rb.insertDecomposed(info.Decomposition()) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
225 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
226 |
rb.insertSingle(src, i, info) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
227 |
return iSuccess |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
228 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
229 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
230 |
// insertUnsafe inserts the given rune in the buffer ordered by CCC. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
231 |
// It is assumed there is sufficient space to hold the runes. It is the |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
232 |
// responsibility of the caller to ensure this. This can be done by checking |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
233 |
// the state returned by the streamSafe type. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
234 |
func (rb *reorderBuffer) insertUnsafe(src input, i int, info Properties) { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
235 |
if rune := src.hangul(i); rune != 0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
236 |
rb.decomposeHangul(rune) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
237 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
238 |
if info.hasDecomposition() { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
239 |
// TODO: inline. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
240 |
rb.insertDecomposed(info.Decomposition()) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
241 |
} else { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
242 |
rb.insertSingle(src, i, info) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
243 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
244 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
245 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
246 |
// insertDecomposed inserts an entry in to the reorderBuffer for each rune |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
247 |
// in dcomp. dcomp must be a sequence of decomposed UTF-8-encoded runes. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
248 |
// It flushes the buffer on each new segment start. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
249 |
func (rb *reorderBuffer) insertDecomposed(dcomp []byte) insertErr { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
250 |
rb.tmpBytes.setBytes(dcomp) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
251 |
// As the streamSafe accounting already handles the counting for modifiers, |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
252 |
// we don't have to call next. However, we do need to keep the accounting |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
253 |
// intact when flushing the buffer. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
254 |
for i := 0; i < len(dcomp); { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
255 |
info := rb.f.info(rb.tmpBytes, i) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
256 |
if info.BoundaryBefore() && rb.nrune > 0 && !rb.doFlush() { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
257 |
return iShortDst |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
258 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
259 |
i += copy(rb.byte[rb.nbyte:], dcomp[i:i+int(info.size)]) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
260 |
rb.insertOrdered(info) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
261 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
262 |
return iSuccess |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
263 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
264 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
265 |
// insertSingle inserts an entry in the reorderBuffer for the rune at |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
266 |
// position i. info is the runeInfo for the rune at position i. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
267 |
func (rb *reorderBuffer) insertSingle(src input, i int, info Properties) { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
268 |
src.copySlice(rb.byte[rb.nbyte:], i, i+int(info.size)) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
269 |
rb.insertOrdered(info) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
270 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
271 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
272 |
// insertCGJ inserts a Combining Grapheme Joiner (0x034f) into rb. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
273 |
func (rb *reorderBuffer) insertCGJ() { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
274 |
rb.insertSingle(input{str: GraphemeJoiner}, 0, Properties{size: uint8(len(GraphemeJoiner))}) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
275 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
276 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
277 |
// appendRune inserts a rune at the end of the buffer. It is used for Hangul. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
278 |
func (rb *reorderBuffer) appendRune(r rune) { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
279 |
bn := rb.nbyte |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
280 |
sz := utf8.EncodeRune(rb.byte[bn:], rune(r)) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
281 |
rb.nbyte += utf8.UTFMax |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
282 |
rb.rune[rb.nrune] = Properties{pos: bn, size: uint8(sz)} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
283 |
rb.nrune++ |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
284 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
285 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
286 |
// assignRune sets a rune at position pos. It is used for Hangul and recomposition. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
287 |
func (rb *reorderBuffer) assignRune(pos int, r rune) { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
288 |
bn := rb.rune[pos].pos |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
289 |
sz := utf8.EncodeRune(rb.byte[bn:], rune(r)) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
290 |
rb.rune[pos] = Properties{pos: bn, size: uint8(sz)} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
291 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
292 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
293 |
// runeAt returns the rune at position n. It is used for Hangul and recomposition. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
294 |
func (rb *reorderBuffer) runeAt(n int) rune { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
295 |
inf := rb.rune[n] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
296 |
r, _ := utf8.DecodeRune(rb.byte[inf.pos : inf.pos+inf.size]) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
297 |
return r |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
298 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
299 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
300 |
// bytesAt returns the UTF-8 encoding of the rune at position n. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
301 |
// It is used for Hangul and recomposition. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
302 |
func (rb *reorderBuffer) bytesAt(n int) []byte { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
303 |
inf := rb.rune[n] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
304 |
return rb.byte[inf.pos : int(inf.pos)+int(inf.size)] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
305 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
306 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
307 |
// For Hangul we combine algorithmically, instead of using tables. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
308 |
const ( |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
309 |
hangulBase = 0xAC00 // UTF-8(hangulBase) -> EA B0 80 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
310 |
hangulBase0 = 0xEA |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
311 |
hangulBase1 = 0xB0 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
312 |
hangulBase2 = 0x80 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
313 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
314 |
hangulEnd = hangulBase + jamoLVTCount // UTF-8(0xD7A4) -> ED 9E A4 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
315 |
hangulEnd0 = 0xED |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
316 |
hangulEnd1 = 0x9E |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
317 |
hangulEnd2 = 0xA4 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
318 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
319 |
jamoLBase = 0x1100 // UTF-8(jamoLBase) -> E1 84 00 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
320 |
jamoLBase0 = 0xE1 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
321 |
jamoLBase1 = 0x84 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
322 |
jamoLEnd = 0x1113 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
323 |
jamoVBase = 0x1161 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
324 |
jamoVEnd = 0x1176 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
325 |
jamoTBase = 0x11A7 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
326 |
jamoTEnd = 0x11C3 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
327 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
328 |
jamoTCount = 28 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
329 |
jamoVCount = 21 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
330 |
jamoVTCount = 21 * 28 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
331 |
jamoLVTCount = 19 * 21 * 28 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
332 |
) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
333 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
334 |
const hangulUTF8Size = 3 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
335 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
336 |
func isHangul(b []byte) bool { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
337 |
if len(b) < hangulUTF8Size { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
338 |
return false |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
339 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
340 |
b0 := b[0] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
341 |
if b0 < hangulBase0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
342 |
return false |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
343 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
344 |
b1 := b[1] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
345 |
switch { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
346 |
case b0 == hangulBase0: |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
347 |
return b1 >= hangulBase1 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
348 |
case b0 < hangulEnd0: |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
349 |
return true |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
350 |
case b0 > hangulEnd0: |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
351 |
return false |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
352 |
case b1 < hangulEnd1: |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
353 |
return true |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
354 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
355 |
return b1 == hangulEnd1 && b[2] < hangulEnd2 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
356 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
357 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
358 |
func isHangulString(b string) bool { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
359 |
if len(b) < hangulUTF8Size { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
360 |
return false |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
361 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
362 |
b0 := b[0] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
363 |
if b0 < hangulBase0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
364 |
return false |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
365 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
366 |
b1 := b[1] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
367 |
switch { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
368 |
case b0 == hangulBase0: |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
369 |
return b1 >= hangulBase1 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
370 |
case b0 < hangulEnd0: |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
371 |
return true |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
372 |
case b0 > hangulEnd0: |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
373 |
return false |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
374 |
case b1 < hangulEnd1: |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
375 |
return true |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
376 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
377 |
return b1 == hangulEnd1 && b[2] < hangulEnd2 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
378 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
379 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
380 |
// Caller must ensure len(b) >= 2. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
381 |
func isJamoVT(b []byte) bool { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
382 |
// True if (rune & 0xff00) == jamoLBase |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
383 |
return b[0] == jamoLBase0 && (b[1]&0xFC) == jamoLBase1 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
384 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
385 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
386 |
func isHangulWithoutJamoT(b []byte) bool { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
387 |
c, _ := utf8.DecodeRune(b) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
388 |
c -= hangulBase |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
389 |
return c < jamoLVTCount && c%jamoTCount == 0 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
390 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
391 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
392 |
// decomposeHangul writes the decomposed Hangul to buf and returns the number |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
393 |
// of bytes written. len(buf) should be at least 9. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
394 |
func decomposeHangul(buf []byte, r rune) int { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
395 |
const JamoUTF8Len = 3 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
396 |
r -= hangulBase |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
397 |
x := r % jamoTCount |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
398 |
r /= jamoTCount |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
399 |
utf8.EncodeRune(buf, jamoLBase+r/jamoVCount) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
400 |
utf8.EncodeRune(buf[JamoUTF8Len:], jamoVBase+r%jamoVCount) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
401 |
if x != 0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
402 |
utf8.EncodeRune(buf[2*JamoUTF8Len:], jamoTBase+x) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
403 |
return 3 * JamoUTF8Len |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
404 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
405 |
return 2 * JamoUTF8Len |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
406 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
407 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
408 |
// decomposeHangul algorithmically decomposes a Hangul rune into |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
409 |
// its Jamo components. |
251 | 410 |
// See https://unicode.org/reports/tr15/#Hangul for details on decomposing Hangul. |
242
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
411 |
func (rb *reorderBuffer) decomposeHangul(r rune) { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
412 |
r -= hangulBase |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
413 |
x := r % jamoTCount |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
414 |
r /= jamoTCount |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
415 |
rb.appendRune(jamoLBase + r/jamoVCount) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
416 |
rb.appendRune(jamoVBase + r%jamoVCount) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
417 |
if x != 0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
418 |
rb.appendRune(jamoTBase + x) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
419 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
420 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
421 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
422 |
// combineHangul algorithmically combines Jamo character components into Hangul. |
251 | 423 |
// See https://unicode.org/reports/tr15/#Hangul for details on combining Hangul. |
242
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
424 |
func (rb *reorderBuffer) combineHangul(s, i, k int) { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
425 |
b := rb.rune[:] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
426 |
bn := rb.nrune |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
427 |
for ; i < bn; i++ { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
428 |
cccB := b[k-1].ccc |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
429 |
cccC := b[i].ccc |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
430 |
if cccB == 0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
431 |
s = k - 1 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
432 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
433 |
if s != k-1 && cccB >= cccC { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
434 |
// b[i] is blocked by greater-equal cccX below it |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
435 |
b[k] = b[i] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
436 |
k++ |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
437 |
} else { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
438 |
l := rb.runeAt(s) // also used to compare to hangulBase |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
439 |
v := rb.runeAt(i) // also used to compare to jamoT |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
440 |
switch { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
441 |
case jamoLBase <= l && l < jamoLEnd && |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
442 |
jamoVBase <= v && v < jamoVEnd: |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
443 |
// 11xx plus 116x to LV |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
444 |
rb.assignRune(s, hangulBase+ |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
445 |
(l-jamoLBase)*jamoVTCount+(v-jamoVBase)*jamoTCount) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
446 |
case hangulBase <= l && l < hangulEnd && |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
447 |
jamoTBase < v && v < jamoTEnd && |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
448 |
((l-hangulBase)%jamoTCount) == 0: |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
449 |
// ACxx plus 11Ax to LVT |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
450 |
rb.assignRune(s, l+v-jamoTBase) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
451 |
default: |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
452 |
b[k] = b[i] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
453 |
k++ |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
454 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
455 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
456 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
457 |
rb.nrune = k |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
458 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
459 |
|
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
460 |
// compose recombines the runes in the buffer. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
461 |
// It should only be used to recompose a single segment, as it will not |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
462 |
// handle alternations between Hangul and non-Hangul characters correctly. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
463 |
func (rb *reorderBuffer) compose() { |
251 | 464 |
// Lazily load the map used by the combine func below, but do |
465 |
// it outside of the loop. |
|
466 |
recompMapOnce.Do(buildRecompMap) |
|
467 |
||
242
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
468 |
// UAX #15, section X5 , including Corrigendum #5 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
469 |
// "In any character sequence beginning with starter S, a character C is |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
470 |
// blocked from S if and only if there is some character B between S |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
471 |
// and C, and either B is a starter or it has the same or higher |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
472 |
// combining class as C." |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
473 |
bn := rb.nrune |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
474 |
if bn == 0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
475 |
return |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
476 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
477 |
k := 1 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
478 |
b := rb.rune[:] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
479 |
for s, i := 0, 1; i < bn; i++ { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
480 |
if isJamoVT(rb.bytesAt(i)) { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
481 |
// Redo from start in Hangul mode. Necessary to support |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
482 |
// U+320E..U+321E in NFKC mode. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
483 |
rb.combineHangul(s, i, k) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
484 |
return |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
485 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
486 |
ii := b[i] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
487 |
// We can only use combineForward as a filter if we later |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
488 |
// get the info for the combined character. This is more |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
489 |
// expensive than using the filter. Using combinesBackward() |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
490 |
// is safe. |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
491 |
if ii.combinesBackward() { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
492 |
cccB := b[k-1].ccc |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
493 |
cccC := ii.ccc |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
494 |
blocked := false // b[i] blocked by starter or greater or equal CCC? |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
495 |
if cccB == 0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
496 |
s = k - 1 |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
497 |
} else { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
498 |
blocked = s != k-1 && cccB >= cccC |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
499 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
500 |
if !blocked { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
501 |
combined := combine(rb.runeAt(s), rb.runeAt(i)) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
502 |
if combined != 0 { |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
503 |
rb.assignRune(s, combined) |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
504 |
continue |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
505 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
506 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
507 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
508 |
b[k] = b[i] |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
509 |
k++ |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
510 |
} |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
511 |
rb.nrune = k |
2a9ec03fe5a1
Use vendoring for backward compatibility
Mikael Berthe <mikael@lilotux.net>
parents:
diff
changeset
|
512 |
} |