letter.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package unicode provides data and functions to test some properties of
     6  // Unicode code points.
     7  package unicode
     8  
     9  const (
    10  	MaxRune         = '\U0010FFFF' // Maximum valid Unicode code point.
    11  	ReplacementChar = '\uFFFD'     // Represents invalid code points.
    12  	MaxASCII        = '\u007F'     // maximum ASCII value.
    13  	MaxLatin1       = '\u00FF'     // maximum Latin-1 value.
    14  )
    15  
    16  // RangeTable defines a set of Unicode code points by listing the ranges of
    17  // code points within the set. The ranges are listed in two slices
    18  // to save space: a slice of 16-bit ranges and a slice of 32-bit ranges.
    19  // The two slices must be in sorted order and non-overlapping.
    20  // Also, R32 should contain only values >= 0x10000 (1<<16).
    21  type RangeTable struct {
    22  	R16         []Range16
    23  	R32         []Range32
    24  	LatinOffset int // number of entries in R16 with Hi <= MaxLatin1
    25  }
    26  
    27  // Range16 represents of a range of 16-bit Unicode code points. The range runs from Lo to Hi
    28  // inclusive and has the specified stride.
    29  type Range16 struct {
    30  	Lo     uint16
    31  	Hi     uint16
    32  	Stride uint16
    33  }
    34  
    35  // Range32 represents of a range of Unicode code points and is used when one or
    36  // more of the values will not fit in 16 bits. The range runs from Lo to Hi
    37  // inclusive and has the specified stride. Lo and Hi must always be >= 1<<16.
    38  type Range32 struct {
    39  	Lo     uint32
    40  	Hi     uint32
    41  	Stride uint32
    42  }
    43  
    44  // CaseRange represents a range of Unicode code points for simple (one
    45  // code point to one code point) case conversion.
    46  // The range runs from Lo to Hi inclusive, with a fixed stride of 1. Deltas
    47  // are the number to add to the code point to reach the code point for a
    48  // different case for that character. They may be negative. If zero, it
    49  // means the character is in the corresponding case. There is a special
    50  // case representing sequences of alternating corresponding Upper and Lower
    51  // pairs. It appears with a fixed Delta of
    52  //
    53  //	{UpperLower, UpperLower, UpperLower}
    54  //
    55  // The constant UpperLower has an otherwise impossible delta value.
    56  type CaseRange struct {
    57  	Lo    uint32
    58  	Hi    uint32
    59  	Delta d
    60  }
    61  
    62  // SpecialCase represents language-specific case mappings such as Turkish.
    63  // Methods of SpecialCase customize (by overriding) the standard mappings.
    64  type SpecialCase []CaseRange
    65  
    66  // BUG(r): There is no mechanism for full case folding, that is, for
    67  // characters that involve multiple runes in the input or output.
    68  
    69  // Indices into the Delta arrays inside CaseRanges for case mapping.
    70  const (
    71  	UpperCase = iota
    72  	LowerCase
    73  	TitleCase
    74  	MaxCase
    75  )
    76  
    77  type d [MaxCase]rune // to make the CaseRanges text shorter
    78  
    79  // If the Delta field of a [CaseRange] is UpperLower, it means
    80  // this CaseRange represents a sequence of the form (say)
    81  // [Upper] [Lower] [Upper] [Lower].
    82  const (
    83  	UpperLower = MaxRune + 1 // (Cannot be a valid delta.)
    84  )
    85  
    86  // linearMax is the maximum size table for linear search for non-Latin1 rune.
    87  // Derived by running 'go test -calibrate'.
    88  const linearMax = 18
    89  
    90  // is16 reports whether r is in the sorted slice of 16-bit ranges.
    91  func is16(ranges []Range16, r uint16) bool {
    92  	if len(ranges) <= linearMax || r <= MaxLatin1 {
    93  		for i := range ranges {
    94  			range_ := &ranges[i]
    95  			if r < range_.Lo {
    96  				return false
    97  			}
    98  			if r <= range_.Hi {
    99  				return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
   100  			}
   101  		}
   102  		return false
   103  	}
   104  
   105  	// binary search over ranges
   106  	lo := 0
   107  	hi := len(ranges)
   108  	for lo < hi {
   109  		m := int(uint(lo+hi) >> 1)
   110  		range_ := &ranges[m]
   111  		if range_.Lo <= r && r <= range_.Hi {
   112  			return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
   113  		}
   114  		if r < range_.Lo {
   115  			hi = m
   116  		} else {
   117  			lo = m + 1
   118  		}
   119  	}
   120  	return false
   121  }
   122  
   123  // is32 reports whether r is in the sorted slice of 32-bit ranges.
   124  func is32(ranges []Range32, r uint32) bool {
   125  	if len(ranges) <= linearMax {
   126  		for i := range ranges {
   127  			range_ := &ranges[i]
   128  			if r < range_.Lo {
   129  				return false
   130  			}
   131  			if r <= range_.Hi {
   132  				return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
   133  			}
   134  		}
   135  		return false
   136  	}
   137  
   138  	// binary search over ranges
   139  	lo := 0
   140  	hi := len(ranges)
   141  	for lo < hi {
   142  		m := int(uint(lo+hi) >> 1)
   143  		range_ := ranges[m]
   144  		if range_.Lo <= r && r <= range_.Hi {
   145  			return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
   146  		}
   147  		if r < range_.Lo {
   148  			hi = m
   149  		} else {
   150  			lo = m + 1
   151  		}
   152  	}
   153  	return false
   154  }
   155  
   156  // Is reports whether the rune is in the specified table of ranges.
   157  func Is(rangeTab *RangeTable, r rune) bool {
   158  	r16 := rangeTab.R16
   159  	// Compare as uint32 to correctly handle negative runes.
   160  	if len(r16) > 0 && uint32(r) <= uint32(r16[len(r16)-1].Hi) {
   161  		return is16(r16, uint16(r))
   162  	}
   163  	r32 := rangeTab.R32
   164  	if len(r32) > 0 && r >= rune(r32[0].Lo) {
   165  		return is32(r32, uint32(r))
   166  	}
   167  	return false
   168  }
   169  
   170  func isExcludingLatin(rangeTab *RangeTable, r rune) bool {
   171  	r16 := rangeTab.R16
   172  	// Compare as uint32 to correctly handle negative runes.
   173  	if off := rangeTab.LatinOffset; len(r16) > off && uint32(r) <= uint32(r16[len(r16)-1].Hi) {
   174  		return is16(r16[off:], uint16(r))
   175  	}
   176  	r32 := rangeTab.R32
   177  	if len(r32) > 0 && r >= rune(r32[0].Lo) {
   178  		return is32(r32, uint32(r))
   179  	}
   180  	return false
   181  }
   182  
   183  // IsUpper reports whether the rune is an upper case letter.
   184  func IsUpper(r rune) bool {
   185  	// See comment in IsGraphic.
   186  	if uint32(r) <= MaxLatin1 {
   187  		return properties[uint8(r)]&pLmask == pLu
   188  	}
   189  	return isExcludingLatin(Upper, r)
   190  }
   191  
   192  // IsLower reports whether the rune is a lower case letter.
   193  func IsLower(r rune) bool {
   194  	// See comment in IsGraphic.
   195  	if uint32(r) <= MaxLatin1 {
   196  		return properties[uint8(r)]&pLmask == pLl
   197  	}
   198  	return isExcludingLatin(Lower, r)
   199  }
   200  
   201  // IsTitle reports whether the rune is a title case letter.
   202  func IsTitle(r rune) bool {
   203  	if r <= MaxLatin1 {
   204  		return false
   205  	}
   206  	return isExcludingLatin(Title, r)
   207  }
   208  
   209  // lookupCaseRange returns the CaseRange mapping for rune r or nil if no
   210  // mapping exists for r.
   211  func lookupCaseRange(r rune, caseRange []CaseRange) *CaseRange {
   212  	// binary search over ranges
   213  	lo := 0
   214  	hi := len(caseRange)
   215  	for lo < hi {
   216  		m := int(uint(lo+hi) >> 1)
   217  		cr := &caseRange[m]
   218  		if rune(cr.Lo) <= r && r <= rune(cr.Hi) {
   219  			return cr
   220  		}
   221  		if r < rune(cr.Lo) {
   222  			hi = m
   223  		} else {
   224  			lo = m + 1
   225  		}
   226  	}
   227  	return nil
   228  }
   229  
   230  // convertCase converts r to _case using CaseRange cr.
   231  func convertCase(_case int, r rune, cr *CaseRange) rune {
   232  	delta := cr.Delta[_case]
   233  	if delta > MaxRune {
   234  		// In an Upper-Lower sequence, which always starts with
   235  		// an UpperCase letter, the real deltas always look like:
   236  		//	{0, 1, 0}    UpperCase (Lower is next)
   237  		//	{-1, 0, -1}  LowerCase (Upper, Title are previous)
   238  		// The characters at even offsets from the beginning of the
   239  		// sequence are upper case; the ones at odd offsets are lower.
   240  		// The correct mapping can be done by clearing or setting the low
   241  		// bit in the sequence offset.
   242  		// The constants UpperCase and TitleCase are even while LowerCase
   243  		// is odd so we take the low bit from _case.
   244  		return rune(cr.Lo) + ((r-rune(cr.Lo))&^1 | rune(_case&1))
   245  	}
   246  	return r + delta
   247  }
   248  
   249  // to maps the rune using the specified case mapping.
   250  // It additionally reports whether caseRange contained a mapping for r.
   251  func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping bool) {
   252  	if _case < 0 || MaxCase <= _case {
   253  		return ReplacementChar, false // as reasonable an error as any
   254  	}
   255  	if cr := lookupCaseRange(r, caseRange); cr != nil {
   256  		return convertCase(_case, r, cr), true
   257  	}
   258  	return r, false
   259  }
   260  
   261  // To maps the rune to the specified case: [UpperCase], [LowerCase], or [TitleCase].
   262  func To(_case int, r rune) rune {
   263  	r, _ = to(_case, r, CaseRanges)
   264  	return r
   265  }
   266  
   267  // ToUpper maps the rune to upper case.
   268  func ToUpper(r rune) rune {
   269  	if r <= MaxASCII {
   270  		if 'a' <= r && r <= 'z' {
   271  			r -= 'a' - 'A'
   272  		}
   273  		return r
   274  	}
   275  	return To(UpperCase, r)
   276  }
   277  
   278  // ToLower maps the rune to lower case.
   279  func ToLower(r rune) rune {
   280  	if r <= MaxASCII {
   281  		if 'A' <= r && r <= 'Z' {
   282  			r += 'a' - 'A'
   283  		}
   284  		return r
   285  	}
   286  	return To(LowerCase, r)
   287  }
   288  
   289  // ToTitle maps the rune to title case.
   290  func ToTitle(r rune) rune {
   291  	if r <= MaxASCII {
   292  		if 'a' <= r && r <= 'z' { // title case is upper case for ASCII
   293  			r -= 'a' - 'A'
   294  		}
   295  		return r
   296  	}
   297  	return To(TitleCase, r)
   298  }
   299  
   300  // ToUpper maps the rune to upper case giving priority to the special mapping.
   301  func (special SpecialCase) ToUpper(r rune) rune {
   302  	r1, hadMapping := to(UpperCase, r, []CaseRange(special))
   303  	if r1 == r && !hadMapping {
   304  		r1 = ToUpper(r)
   305  	}
   306  	return r1
   307  }
   308  
   309  // ToTitle maps the rune to title case giving priority to the special mapping.
   310  func (special SpecialCase) ToTitle(r rune) rune {
   311  	r1, hadMapping := to(TitleCase, r, []CaseRange(special))
   312  	if r1 == r && !hadMapping {
   313  		r1 = ToTitle(r)
   314  	}
   315  	return r1
   316  }
   317  
   318  // ToLower maps the rune to lower case giving priority to the special mapping.
   319  func (special SpecialCase) ToLower(r rune) rune {
   320  	r1, hadMapping := to(LowerCase, r, []CaseRange(special))
   321  	if r1 == r && !hadMapping {
   322  		r1 = ToLower(r)
   323  	}
   324  	return r1
   325  }
   326  
   327  // caseOrbit is defined in tables.go as []foldPair. Right now all the
   328  // entries fit in uint16, so use uint16. If that changes, compilation
   329  // will fail (the constants in the composite literal will not fit in uint16)
   330  // and the types here can change to uint32.
   331  type foldPair struct {
   332  	From uint16
   333  	To   uint16
   334  }
   335  
   336  // SimpleFold iterates over Unicode code points equivalent under
   337  // the Unicode-defined simple case folding. Among the code points
   338  // equivalent to rune (including rune itself), SimpleFold returns the
   339  // smallest rune > r if one exists, or else the smallest rune >= 0.
   340  // If r is not a valid Unicode code point, SimpleFold(r) returns r.
   341  //
   342  // For example:
   343  //
   344  //	SimpleFold('A') = 'a'
   345  //	SimpleFold('a') = 'A'
   346  //
   347  //	SimpleFold('K') = 'k'
   348  //	SimpleFold('k') = '\u212A' (Kelvin symbol, K)
   349  //	SimpleFold('\u212A') = 'K'
   350  //
   351  //	SimpleFold('1') = '1'
   352  //
   353  //	SimpleFold(-2) = -2
   354  func SimpleFold(r rune) rune {
   355  	if r < 0 || r > MaxRune {
   356  		return r
   357  	}
   358  
   359  	if int(r) < len(asciiFold) {
   360  		return rune(asciiFold[r])
   361  	}
   362  
   363  	// Consult caseOrbit table for special cases.
   364  	lo := 0
   365  	hi := len(caseOrbit)
   366  	for lo < hi {
   367  		m := int(uint(lo+hi) >> 1)
   368  		if rune(caseOrbit[m].From) < r {
   369  			lo = m + 1
   370  		} else {
   371  			hi = m
   372  		}
   373  	}
   374  	if lo < len(caseOrbit) && rune(caseOrbit[lo].From) == r {
   375  		return rune(caseOrbit[lo].To)
   376  	}
   377  
   378  	// No folding specified. This is a one- or two-element
   379  	// equivalence class containing rune and ToLower(rune)
   380  	// and ToUpper(rune) if they are different from rune.
   381  	if cr := lookupCaseRange(r, CaseRanges); cr != nil {
   382  		if l := convertCase(LowerCase, r, cr); l != r {
   383  			return l
   384  		}
   385  		return convertCase(UpperCase, r, cr)
   386  	}
   387  	return r
   388  }
   389
View as plain text