1 // Copyright 2014 The Go Authors. All rights reserved.

     2 // Use of this source code is governed by a BSD-style

     3 // license that can be found in the LICENSE file.

     4 

     5 // Package runes provide transforms for UTF-8 encoded text.

     6 package runes // import "golang.org/x/text/runes"

     7 

     8 import (

     9 	"unicode"

    10 	"unicode/utf8"

    11 

    12 	"golang.org/x/text/transform"

    13 )

    14 

    15 // A Set is a collection of runes.

    16 type Set interface {

    17 	// Contains returns true if r is contained in the set.

    18 	Contains(r rune) bool

    19 }

    20 

    21 type setFunc func(rune) bool

    22 

    23 func (s setFunc) Contains(r rune) bool {

    24 	return s(r)

    25 }

    26 

    27 // Note: using funcs here instead of wrapping types result in cleaner

    28 // documentation and a smaller API.

    29 

    30 // In creates a Set with a Contains method that returns true for all runes in

    31 // the given RangeTable.

    32 func In(rt *unicode.RangeTable) Set {

    33 	return setFunc(func(r rune) bool { return unicode.Is(rt, r) })

    34 }

    35 

    36 // In creates a Set with a Contains method that returns true for all runes not

    37 // in the given RangeTable.

    38 func NotIn(rt *unicode.RangeTable) Set {

    39 	return setFunc(func(r rune) bool { return !unicode.Is(rt, r) })

    40 }

    41 

    42 // Predicate creates a Set with a Contains method that returns f(r).

    43 func Predicate(f func(rune) bool) Set {

    44 	return setFunc(f)

    45 }

    46 

    47 // Transformer implements the transform.Transformer interface.

    48 type Transformer struct {

    49 	t transform.SpanningTransformer

    50 }

    51 

    52 func (t Transformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {

    53 	return t.t.Transform(dst, src, atEOF)

    54 }

    55 

    56 func (t Transformer) Span(b []byte, atEOF bool) (n int, err error) {

    57 	return t.t.Span(b, atEOF)

    58 }

    59 

    60 func (t Transformer) Reset() { t.t.Reset() }

    61 

    62 // Bytes returns a new byte slice with the result of converting b using t.  It

    63 // calls Reset on t. It returns nil if any error was found. This can only happen

    64 // if an error-producing Transformer is passed to If.

    65 func (t Transformer) Bytes(b []byte) []byte {

    66 	b, _, err := transform.Bytes(t, b)

    67 	if err != nil {

    68 		return nil

    69 	}

    70 	return b

    71 }

    72 

    73 // String returns a string with the result of converting s using t. It calls

    74 // Reset on t. It returns the empty string if any error was found. This can only

    75 // happen if an error-producing Transformer is passed to If.

    76 func (t Transformer) String(s string) string {

    77 	s, _, err := transform.String(t, s)

    78 	if err != nil {

    79 		return ""

    80 	}

    81 	return s

    82 }

    83 

    84 // TODO:

    85 // - Copy: copying strings and bytes in whole-rune units.

    86 // - Validation (maybe)

    87 // - Well-formed-ness (maybe)

    88 

    89 const runeErrorString = string(utf8.RuneError)

    90 

    91 // Remove returns a Transformer that removes runes r for which s.Contains(r).

    92 // Illegal input bytes are replaced by RuneError before being passed to f.

    93 func Remove(s Set) Transformer {

    94 	if f, ok := s.(setFunc); ok {

    95 		// This little trick cuts the running time of BenchmarkRemove for sets

    96 		// created by Predicate roughly in half.

    97 		// TODO: special-case RangeTables as well.

    98 		return Transformer{remove(f)}

    99 	}

   100 	return Transformer{remove(s.Contains)}

   101 }

   102 

   103 // TODO: remove transform.RemoveFunc.

   104 

   105 type remove func(r rune) bool

   106 

   107 func (remove) Reset() {}

   108 

   109 // Span implements transform.Spanner.

   110 func (t remove) Span(src []byte, atEOF bool) (n int, err error) {

   111 	for r, size := rune(0), 0; n < len(src); {

   112 		if r = rune(src[n]); r < utf8.RuneSelf {

   113 			size = 1

   114 		} else if r, size = utf8.DecodeRune(src[n:]); size == 1 {

   115 			// Invalid rune.

   116 			if !atEOF && !utf8.FullRune(src[n:]) {

   117 				err = transform.ErrShortSrc

   118 			} else {

   119 				err = transform.ErrEndOfSpan

   120 			}

   121 			break

   122 		}

   123 		if t(r) {

   124 			err = transform.ErrEndOfSpan

   125 			break

   126 		}

   127 		n += size

   128 	}

   129 	return

   130 }

   131 

   132 // Transform implements transform.Transformer.

   133 func (t remove) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {

   134 	for r, size := rune(0), 0; nSrc < len(src); {

   135 		if r = rune(src[nSrc]); r < utf8.RuneSelf {

   136 			size = 1

   137 		} else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 {

   138 			// Invalid rune.

   139 			if !atEOF && !utf8.FullRune(src[nSrc:]) {

   140 				err = transform.ErrShortSrc

   141 				break

   142 			}

   143 			// We replace illegal bytes with RuneError. Not doing so might

   144 			// otherwise turn a sequence of invalid UTF-8 into valid UTF-8.

   145 			// The resulting byte sequence may subsequently contain runes

   146 			// for which t(r) is true that were passed unnoticed.

   147 			if !t(utf8.RuneError) {

   148 				if nDst+3 > len(dst) {

   149 					err = transform.ErrShortDst

   150 					break

   151 				}

   152 				dst[nDst+0] = runeErrorString[0]

   153 				dst[nDst+1] = runeErrorString[1]

   154 				dst[nDst+2] = runeErrorString[2]

   155 				nDst += 3

   156 			}

   157 			nSrc++

   158 			continue

   159 		}

   160 		if t(r) {

   161 			nSrc += size

   162 			continue

   163 		}

   164 		if nDst+size > len(dst) {

   165 			err = transform.ErrShortDst

   166 			break

   167 		}

   168 		for i := 0; i < size; i++ {

   169 			dst[nDst] = src[nSrc]

   170 			nDst++

   171 			nSrc++

   172 		}

   173 	}

   174 	return

   175 }

   176 

   177 // Map returns a Transformer that maps the runes in the input using the given

   178 // mapping. Illegal bytes in the input are converted to utf8.RuneError before

   179 // being passed to the mapping func.

   180 func Map(mapping func(rune) rune) Transformer {

   181 	return Transformer{mapper(mapping)}

   182 }

   183 

   184 type mapper func(rune) rune

   185 

   186 func (mapper) Reset() {}

   187 

   188 // Span implements transform.Spanner.

   189 func (t mapper) Span(src []byte, atEOF bool) (n int, err error) {

   190 	for r, size := rune(0), 0; n < len(src); n += size {

   191 		if r = rune(src[n]); r < utf8.RuneSelf {

   192 			size = 1

   193 		} else if r, size = utf8.DecodeRune(src[n:]); size == 1 {

   194 			// Invalid rune.

   195 			if !atEOF && !utf8.FullRune(src[n:]) {

   196 				err = transform.ErrShortSrc

   197 			} else {

   198 				err = transform.ErrEndOfSpan

   199 			}

   200 			break

   201 		}

   202 		if t(r) != r {

   203 			err = transform.ErrEndOfSpan

   204 			break

   205 		}

   206 	}

   207 	return n, err

   208 }

   209 

   210 // Transform implements transform.Transformer.

   211 func (t mapper) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {

   212 	var replacement rune

   213 	var b [utf8.UTFMax]byte

   214 

   215 	for r, size := rune(0), 0; nSrc < len(src); {

   216 		if r = rune(src[nSrc]); r < utf8.RuneSelf {

   217 			if replacement = t(r); replacement < utf8.RuneSelf {

   218 				if nDst == len(dst) {

   219 					err = transform.ErrShortDst

   220 					break

   221 				}

   222 				dst[nDst] = byte(replacement)

   223 				nDst++

   224 				nSrc++

   225 				continue

   226 			}

   227 			size = 1

   228 		} else if r, size = utf8.DecodeRune(src[nSrc:]); size == 1 {

   229 			// Invalid rune.

   230 			if !atEOF && !utf8.FullRune(src[nSrc:]) {

   231 				err = transform.ErrShortSrc

   232 				break

   233 			}

   234 

   235 			if replacement = t(utf8.RuneError); replacement == utf8.RuneError {

   236 				if nDst+3 > len(dst) {

   237 					err = transform.ErrShortDst

   238 					break

   239 				}

   240 				dst[nDst+0] = runeErrorString[0]

   241 				dst[nDst+1] = runeErrorString[1]

   242 				dst[nDst+2] = runeErrorString[2]

   243 				nDst += 3

   244 				nSrc++

   245 				continue

   246 			}

   247 		} else if replacement = t(r); replacement == r {

   248 			if nDst+size > len(dst) {

   249 				err = transform.ErrShortDst

   250 				break

   251 			}

   252 			for i := 0; i < size; i++ {

   253 				dst[nDst] = src[nSrc]

   254 				nDst++

   255 				nSrc++

   256 			}

   257 			continue

   258 		}

   259 

   260 		n := utf8.EncodeRune(b[:], replacement)

   261 

   262 		if nDst+n > len(dst) {

   263 			err = transform.ErrShortDst

   264 			break

   265 		}

   266 		for i := 0; i < n; i++ {

   267 			dst[nDst] = b[i]

   268 			nDst++

   269 		}

   270 		nSrc += size

   271 	}

   272 	return

   273 }

   274 

   275 // ReplaceIllFormed returns a transformer that replaces all input bytes that are

   276 // not part of a well-formed UTF-8 code sequence with utf8.RuneError.

   277 func ReplaceIllFormed() Transformer {

   278 	return Transformer{&replaceIllFormed{}}

   279 }

   280 

   281 type replaceIllFormed struct{ transform.NopResetter }

   282 

   283 func (t replaceIllFormed) Span(src []byte, atEOF bool) (n int, err error) {

   284 	for n < len(src) {

   285 		// ASCII fast path.

   286 		if src[n] < utf8.RuneSelf {

   287 			n++

   288 			continue

   289 		}

   290 

   291 		r, size := utf8.DecodeRune(src[n:])

   292 

   293 		// Look for a valid non-ASCII rune.

   294 		if r != utf8.RuneError || size != 1 {

   295 			n += size

   296 			continue

   297 		}

   298 

   299 		// Look for short source data.

   300 		if !atEOF && !utf8.FullRune(src[n:]) {

   301 			err = transform.ErrShortSrc

   302 			break

   303 		}

   304 

   305 		// We have an invalid rune.

   306 		err = transform.ErrEndOfSpan

   307 		break

   308 	}

   309 	return n, err

   310 }

   311 

   312 func (t replaceIllFormed) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {

   313 	for nSrc < len(src) {

   314 		// ASCII fast path.

   315 		if r := src[nSrc]; r < utf8.RuneSelf {

   316 			if nDst == len(dst) {

   317 				err = transform.ErrShortDst

   318 				break

   319 			}

   320 			dst[nDst] = r

   321 			nDst++

   322 			nSrc++

   323 			continue

   324 		}

   325 

   326 		// Look for a valid non-ASCII rune.

   327 		if _, size := utf8.DecodeRune(src[nSrc:]); size != 1 {

   328 			if size != copy(dst[nDst:], src[nSrc:nSrc+size]) {

   329 				err = transform.ErrShortDst

   330 				break

   331 			}

   332 			nDst += size

   333 			nSrc += size

   334 			continue

   335 		}

   336 

   337 		// Look for short source data.

   338 		if !atEOF && !utf8.FullRune(src[nSrc:]) {

   339 			err = transform.ErrShortSrc

   340 			break

   341 		}

   342 

   343 		// We have an invalid rune.

   344 		if nDst+3 > len(dst) {

   345 			err = transform.ErrShortDst

   346 			break

   347 		}

   348 		dst[nDst+0] = runeErrorString[0]

   349 		dst[nDst+1] = runeErrorString[1]

   350 		dst[nDst+2] = runeErrorString[2]

   351 		nDst += 3

   352 		nSrc++

   353 	}

   354 	return nDst, nSrc, err

   355 }