language.go

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:generate go run gen.go gen_common.go -output tables.go
     6  
     7  package language // import "golang.org/x/text/internal/language"
     8  
     9  // TODO: Remove above NOTE after:
    10  // - verifying that tables are dropped correctly (most notably matcher tables).
    11  
    12  import (
    13  	"errors"
    14  	"fmt"
    15  	"strings"
    16  )
    17  
    18  const (
    19  	// maxCoreSize is the maximum size of a BCP 47 tag without variants and
    20  	// extensions. Equals max lang (3) + script (4) + max reg (3) + 2 dashes.
    21  	maxCoreSize = 12
    22  
    23  	// max99thPercentileSize is a somewhat arbitrary buffer size that presumably
    24  	// is large enough to hold at least 99% of the BCP 47 tags.
    25  	max99thPercentileSize = 32
    26  
    27  	// maxSimpleUExtensionSize is the maximum size of a -u extension with one
    28  	// key-type pair. Equals len("-u-") + key (2) + dash + max value (8).
    29  	maxSimpleUExtensionSize = 14
    30  )
    31  
    32  // Tag represents a BCP 47 language tag. It is used to specify an instance of a
    33  // specific language or locale. All language tag values are guaranteed to be
    34  // well-formed. The zero value of Tag is Und.
    35  type Tag struct {
    36  	// TODO: the following fields have the form TagTypeID. This name is chosen
    37  	// to allow refactoring the public package without conflicting with its
    38  	// Base, Script, and Region methods. Once the transition is fully completed
    39  	// the ID can be stripped from the name.
    40  
    41  	LangID   Language
    42  	RegionID Region
    43  	// TODO: we will soon run out of positions for ScriptID. Idea: instead of
    44  	// storing lang, region, and ScriptID codes, store only the compact index and
    45  	// have a lookup table from this code to its expansion. This greatly speeds
    46  	// up table lookup, speed up common variant cases.
    47  	// This will also immediately free up 3 extra bytes. Also, the pVariant
    48  	// field can now be moved to the lookup table, as the compact index uniquely
    49  	// determines the offset of a possible variant.
    50  	ScriptID Script
    51  	pVariant byte   // offset in str, includes preceding '-'
    52  	pExt     uint16 // offset of first extension, includes preceding '-'
    53  
    54  	// str is the string representation of the Tag. It will only be used if the
    55  	// tag has variants or extensions.
    56  	str string
    57  }
    58  
    59  // Make is a convenience wrapper for Parse that omits the error.
    60  // In case of an error, a sensible default is returned.
    61  func Make(s string) Tag {
    62  	t, _ := Parse(s)
    63  	return t
    64  }
    65  
    66  // Raw returns the raw base language, script and region, without making an
    67  // attempt to infer their values.
    68  // TODO: consider removing
    69  func (t Tag) Raw() (b Language, s Script, r Region) {
    70  	return t.LangID, t.ScriptID, t.RegionID
    71  }
    72  
    73  // equalTags compares language, script and region subtags only.
    74  func (t Tag) equalTags(a Tag) bool {
    75  	return t.LangID == a.LangID && t.ScriptID == a.ScriptID && t.RegionID == a.RegionID
    76  }
    77  
    78  // IsRoot returns true if t is equal to language "und".
    79  func (t Tag) IsRoot() bool {
    80  	if int(t.pVariant) < len(t.str) {
    81  		return false
    82  	}
    83  	return t.equalTags(Und)
    84  }
    85  
    86  // IsPrivateUse reports whether the Tag consists solely of an IsPrivateUse use
    87  // tag.
    88  func (t Tag) IsPrivateUse() bool {
    89  	return t.str != "" && t.pVariant == 0
    90  }
    91  
    92  // RemakeString is used to update t.str in case lang, script or region changed.
    93  // It is assumed that pExt and pVariant still point to the start of the
    94  // respective parts.
    95  func (t *Tag) RemakeString() {
    96  	if t.str == "" {
    97  		return
    98  	}
    99  	extra := t.str[t.pVariant:]
   100  	if t.pVariant > 0 {
   101  		extra = extra[1:]
   102  	}
   103  	if t.equalTags(Und) && strings.HasPrefix(extra, "x-") {
   104  		t.str = extra
   105  		t.pVariant = 0
   106  		t.pExt = 0
   107  		return
   108  	}
   109  	var buf [max99thPercentileSize]byte // avoid extra memory allocation in most cases.
   110  	b := buf[:t.genCoreBytes(buf[:])]
   111  	if extra != "" {
   112  		diff := len(b) - int(t.pVariant)
   113  		b = append(b, '-')
   114  		b = append(b, extra...)
   115  		t.pVariant = uint8(int(t.pVariant) + diff)
   116  		t.pExt = uint16(int(t.pExt) + diff)
   117  	} else {
   118  		t.pVariant = uint8(len(b))
   119  		t.pExt = uint16(len(b))
   120  	}
   121  	t.str = string(b)
   122  }
   123  
   124  // genCoreBytes writes a string for the base languages, script and region tags
   125  // to the given buffer and returns the number of bytes written. It will never
   126  // write more than maxCoreSize bytes.
   127  func (t *Tag) genCoreBytes(buf []byte) int {
   128  	n := t.LangID.StringToBuf(buf[:])
   129  	if t.ScriptID != 0 {
   130  		n += copy(buf[n:], "-")
   131  		n += copy(buf[n:], t.ScriptID.String())
   132  	}
   133  	if t.RegionID != 0 {
   134  		n += copy(buf[n:], "-")
   135  		n += copy(buf[n:], t.RegionID.String())
   136  	}
   137  	return n
   138  }
   139  
   140  // String returns the canonical string representation of the language tag.
   141  func (t Tag) String() string {
   142  	if t.str != "" {
   143  		return t.str
   144  	}
   145  	if t.ScriptID == 0 && t.RegionID == 0 {
   146  		return t.LangID.String()
   147  	}
   148  	buf := [maxCoreSize]byte{}
   149  	return string(buf[:t.genCoreBytes(buf[:])])
   150  }
   151  
   152  // MarshalText implements encoding.TextMarshaler.
   153  func (t Tag) MarshalText() (text []byte, err error) {
   154  	if t.str != "" {
   155  		text = append(text, t.str...)
   156  	} else if t.ScriptID == 0 && t.RegionID == 0 {
   157  		text = append(text, t.LangID.String()...)
   158  	} else {
   159  		buf := [maxCoreSize]byte{}
   160  		text = buf[:t.genCoreBytes(buf[:])]
   161  	}
   162  	return text, nil
   163  }
   164  
   165  // UnmarshalText implements encoding.TextUnmarshaler.
   166  func (t *Tag) UnmarshalText(text []byte) error {
   167  	tag, err := Parse(string(text))
   168  	*t = tag
   169  	return err
   170  }
   171  
   172  // Variants returns the part of the tag holding all variants or the empty string
   173  // if there are no variants defined.
   174  func (t Tag) Variants() string {
   175  	if t.pVariant == 0 {
   176  		return ""
   177  	}
   178  	return t.str[t.pVariant:t.pExt]
   179  }
   180  
   181  // VariantOrPrivateUseTags returns variants or private use tags.
   182  func (t Tag) VariantOrPrivateUseTags() string {
   183  	if t.pExt > 0 {
   184  		return t.str[t.pVariant:t.pExt]
   185  	}
   186  	return t.str[t.pVariant:]
   187  }
   188  
   189  // HasString reports whether this tag defines more than just the raw
   190  // components.
   191  func (t Tag) HasString() bool {
   192  	return t.str != ""
   193  }
   194  
   195  // Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
   196  // specific language are substituted with fields from the parent language.
   197  // The parent for a language may change for newer versions of CLDR.
   198  func (t Tag) Parent() Tag {
   199  	if t.str != "" {
   200  		// Strip the variants and extensions.
   201  		b, s, r := t.Raw()
   202  		t = Tag{LangID: b, ScriptID: s, RegionID: r}
   203  		if t.RegionID == 0 && t.ScriptID != 0 && t.LangID != 0 {
   204  			base, _ := addTags(Tag{LangID: t.LangID})
   205  			if base.ScriptID == t.ScriptID {
   206  				return Tag{LangID: t.LangID}
   207  			}
   208  		}
   209  		return t
   210  	}
   211  	if t.LangID != 0 {
   212  		if t.RegionID != 0 {
   213  			maxScript := t.ScriptID
   214  			if maxScript == 0 {
   215  				max, _ := addTags(t)
   216  				maxScript = max.ScriptID
   217  			}
   218  
   219  			for i := range parents {
   220  				if Language(parents[i].lang) == t.LangID && Script(parents[i].maxScript) == maxScript {
   221  					for _, r := range parents[i].fromRegion {
   222  						if Region(r) == t.RegionID {
   223  							return Tag{
   224  								LangID:   t.LangID,
   225  								ScriptID: Script(parents[i].script),
   226  								RegionID: Region(parents[i].toRegion),
   227  							}
   228  						}
   229  					}
   230  				}
   231  			}
   232  
   233  			// Strip the script if it is the default one.
   234  			base, _ := addTags(Tag{LangID: t.LangID})
   235  			if base.ScriptID != maxScript {
   236  				return Tag{LangID: t.LangID, ScriptID: maxScript}
   237  			}
   238  			return Tag{LangID: t.LangID}
   239  		} else if t.ScriptID != 0 {
   240  			// The parent for an base-script pair with a non-default script is
   241  			// "und" instead of the base language.
   242  			base, _ := addTags(Tag{LangID: t.LangID})
   243  			if base.ScriptID != t.ScriptID {
   244  				return Und
   245  			}
   246  			return Tag{LangID: t.LangID}
   247  		}
   248  	}
   249  	return Und
   250  }
   251  
   252  // ParseExtension parses s as an extension and returns it on success.
   253  func ParseExtension(s string) (ext string, err error) {
   254  	defer func() {
   255  		if recover() != nil {
   256  			ext = ""
   257  			err = ErrSyntax
   258  		}
   259  	}()
   260  
   261  	scan := makeScannerString(s)
   262  	var end int
   263  	if n := len(scan.token); n != 1 {
   264  		return "", ErrSyntax
   265  	}
   266  	scan.toLower(0, len(scan.b))
   267  	end = parseExtension(&scan)
   268  	if end != len(s) {
   269  		return "", ErrSyntax
   270  	}
   271  	return string(scan.b), nil
   272  }
   273  
   274  // HasVariants reports whether t has variants.
   275  func (t Tag) HasVariants() bool {
   276  	return uint16(t.pVariant) < t.pExt
   277  }
   278  
   279  // HasExtensions reports whether t has extensions.
   280  func (t Tag) HasExtensions() bool {
   281  	return int(t.pExt) < len(t.str)
   282  }
   283  
   284  // Extension returns the extension of type x for tag t. It will return
   285  // false for ok if t does not have the requested extension. The returned
   286  // extension will be invalid in this case.
   287  func (t Tag) Extension(x byte) (ext string, ok bool) {
   288  	for i := int(t.pExt); i < len(t.str)-1; {
   289  		var ext string
   290  		i, ext = getExtension(t.str, i)
   291  		if ext[0] == x {
   292  			return ext, true
   293  		}
   294  	}
   295  	return "", false
   296  }
   297  
   298  // Extensions returns all extensions of t.
   299  func (t Tag) Extensions() []string {
   300  	e := []string{}
   301  	for i := int(t.pExt); i < len(t.str)-1; {
   302  		var ext string
   303  		i, ext = getExtension(t.str, i)
   304  		e = append(e, ext)
   305  	}
   306  	return e
   307  }
   308  
   309  // TypeForKey returns the type associated with the given key, where key and type
   310  // are of the allowed values defined for the Unicode locale extension ('u') in
   311  // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
   312  // TypeForKey will traverse the inheritance chain to get the correct value.
   313  //
   314  // If there are multiple types associated with a key, only the first will be
   315  // returned. If there is no type associated with a key, it returns the empty
   316  // string.
   317  func (t Tag) TypeForKey(key string) string {
   318  	if _, start, end, _ := t.findTypeForKey(key); end != start {
   319  		s := t.str[start:end]
   320  		if p := strings.IndexByte(s, '-'); p >= 0 {
   321  			s = s[:p]
   322  		}
   323  		return s
   324  	}
   325  	return ""
   326  }
   327  
   328  var (
   329  	errPrivateUse       = errors.New("cannot set a key on a private use tag")
   330  	errInvalidArguments = errors.New("invalid key or type")
   331  )
   332  
   333  // SetTypeForKey returns a new Tag with the key set to type, where key and type
   334  // are of the allowed values defined for the Unicode locale extension ('u') in
   335  // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
   336  // An empty value removes an existing pair with the same key.
   337  func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
   338  	if t.IsPrivateUse() {
   339  		return t, errPrivateUse
   340  	}
   341  	if len(key) != 2 {
   342  		return t, errInvalidArguments
   343  	}
   344  
   345  	// Remove the setting if value is "".
   346  	if value == "" {
   347  		start, sep, end, _ := t.findTypeForKey(key)
   348  		if start != sep {
   349  			// Remove a possible empty extension.
   350  			switch {
   351  			case t.str[start-2] != '-': // has previous elements.
   352  			case end == len(t.str), // end of string
   353  				end+2 < len(t.str) && t.str[end+2] == '-': // end of extension
   354  				start -= 2
   355  			}
   356  			if start == int(t.pVariant) && end == len(t.str) {
   357  				t.str = ""
   358  				t.pVariant, t.pExt = 0, 0
   359  			} else {
   360  				t.str = fmt.Sprintf("%s%s", t.str[:start], t.str[end:])
   361  			}
   362  		}
   363  		return t, nil
   364  	}
   365  
   366  	if len(value) < 3 || len(value) > 8 {
   367  		return t, errInvalidArguments
   368  	}
   369  
   370  	var (
   371  		buf    [maxCoreSize + maxSimpleUExtensionSize]byte
   372  		uStart int // start of the -u extension.
   373  	)
   374  
   375  	// Generate the tag string if needed.
   376  	if t.str == "" {
   377  		uStart = t.genCoreBytes(buf[:])
   378  		buf[uStart] = '-'
   379  		uStart++
   380  	}
   381  
   382  	// Create new key-type pair and parse it to verify.
   383  	b := buf[uStart:]
   384  	copy(b, "u-")
   385  	copy(b[2:], key)
   386  	b[4] = '-'
   387  	b = b[:5+copy(b[5:], value)]
   388  	scan := makeScanner(b)
   389  	if parseExtensions(&scan); scan.err != nil {
   390  		return t, scan.err
   391  	}
   392  
   393  	// Assemble the replacement string.
   394  	if t.str == "" {
   395  		t.pVariant, t.pExt = byte(uStart-1), uint16(uStart-1)
   396  		t.str = string(buf[:uStart+len(b)])
   397  	} else {
   398  		s := t.str
   399  		start, sep, end, hasExt := t.findTypeForKey(key)
   400  		if start == sep {
   401  			if hasExt {
   402  				b = b[2:]
   403  			}
   404  			t.str = fmt.Sprintf("%s-%s%s", s[:sep], b, s[end:])
   405  		} else {
   406  			t.str = fmt.Sprintf("%s-%s%s", s[:start+3], value, s[end:])
   407  		}
   408  	}
   409  	return t, nil
   410  }
   411  
   412  // findTypeForKey returns the start and end position for the type corresponding
   413  // to key or the point at which to insert the key-value pair if the type
   414  // wasn't found. The hasExt return value reports whether an -u extension was present.
   415  // Note: the extensions are typically very small and are likely to contain
   416  // only one key-type pair.
   417  func (t Tag) findTypeForKey(key string) (start, sep, end int, hasExt bool) {
   418  	p := int(t.pExt)
   419  	if len(key) != 2 || p == len(t.str) || p == 0 {
   420  		return p, p, p, false
   421  	}
   422  	s := t.str
   423  
   424  	// Find the correct extension.
   425  	for p++; s[p] != 'u'; p++ {
   426  		if s[p] > 'u' {
   427  			p--
   428  			return p, p, p, false
   429  		}
   430  		if p = nextExtension(s, p); p == len(s) {
   431  			return len(s), len(s), len(s), false
   432  		}
   433  	}
   434  	// Proceed to the hyphen following the extension name.
   435  	p++
   436  
   437  	// curKey is the key currently being processed.
   438  	curKey := ""
   439  
   440  	// Iterate over keys until we get the end of a section.
   441  	for {
   442  		end = p
   443  		for p++; p < len(s) && s[p] != '-'; p++ {
   444  		}
   445  		n := p - end - 1
   446  		if n <= 2 && curKey == key {
   447  			if sep < end {
   448  				sep++
   449  			}
   450  			return start, sep, end, true
   451  		}
   452  		switch n {
   453  		case 0, // invalid string
   454  			1: // next extension
   455  			return end, end, end, true
   456  		case 2:
   457  			// next key
   458  			curKey = s[end+1 : p]
   459  			if curKey > key {
   460  				return end, end, end, true
   461  			}
   462  			start = end
   463  			sep = p
   464  		}
   465  	}
   466  }
   467  
   468  // ParseBase parses a 2- or 3-letter ISO 639 code.
   469  // It returns a ValueError if s is a well-formed but unknown language identifier
   470  // or another error if another error occurred.
   471  func ParseBase(s string) (l Language, err error) {
   472  	defer func() {
   473  		if recover() != nil {
   474  			l = 0
   475  			err = ErrSyntax
   476  		}
   477  	}()
   478  
   479  	if n := len(s); n < 2 || 3 < n {
   480  		return 0, ErrSyntax
   481  	}
   482  	var buf [3]byte
   483  	return getLangID(buf[:copy(buf[:], s)])
   484  }
   485  
   486  // ParseScript parses a 4-letter ISO 15924 code.
   487  // It returns a ValueError if s is a well-formed but unknown script identifier
   488  // or another error if another error occurred.
   489  func ParseScript(s string) (scr Script, err error) {
   490  	defer func() {
   491  		if recover() != nil {
   492  			scr = 0
   493  			err = ErrSyntax
   494  		}
   495  	}()
   496  
   497  	if len(s) != 4 {
   498  		return 0, ErrSyntax
   499  	}
   500  	var buf [4]byte
   501  	return getScriptID(script, buf[:copy(buf[:], s)])
   502  }
   503  
   504  // EncodeM49 returns the Region for the given UN M.49 code.
   505  // It returns an error if r is not a valid code.
   506  func EncodeM49(r int) (Region, error) {
   507  	return getRegionM49(r)
   508  }
   509  
   510  // ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
   511  // It returns a ValueError if s is a well-formed but unknown region identifier
   512  // or another error if another error occurred.
   513  func ParseRegion(s string) (r Region, err error) {
   514  	defer func() {
   515  		if recover() != nil {
   516  			r = 0
   517  			err = ErrSyntax
   518  		}
   519  	}()
   520  
   521  	if n := len(s); n < 2 || 3 < n {
   522  		return 0, ErrSyntax
   523  	}
   524  	var buf [3]byte
   525  	return getRegionID(buf[:copy(buf[:], s)])
   526  }
   527  
   528  // IsCountry returns whether this region is a country or autonomous area. This
   529  // includes non-standard definitions from CLDR.
   530  func (r Region) IsCountry() bool {
   531  	if r == 0 || r.IsGroup() || r.IsPrivateUse() && r != _XK {
   532  		return false
   533  	}
   534  	return true
   535  }
   536  
   537  // IsGroup returns whether this region defines a collection of regions. This
   538  // includes non-standard definitions from CLDR.
   539  func (r Region) IsGroup() bool {
   540  	if r == 0 {
   541  		return false
   542  	}
   543  	return int(regionInclusion[r]) < len(regionContainment)
   544  }
   545  
   546  // Contains returns whether Region c is contained by Region r. It returns true
   547  // if c == r.
   548  func (r Region) Contains(c Region) bool {
   549  	if r == c {
   550  		return true
   551  	}
   552  	g := regionInclusion[r]
   553  	if g >= nRegionGroups {
   554  		return false
   555  	}
   556  	m := regionContainment[g]
   557  
   558  	d := regionInclusion[c]
   559  	b := regionInclusionBits[d]
   560  
   561  	// A contained country may belong to multiple disjoint groups. Matching any
   562  	// of these indicates containment. If the contained region is a group, it
   563  	// must strictly be a subset.
   564  	if d >= nRegionGroups {
   565  		return b&m != 0
   566  	}
   567  	return b&^m == 0
   568  }
   569  
   570  var errNoTLD = errors.New("language: region is not a valid ccTLD")
   571  
   572  // TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
   573  // In all other cases it returns either the region itself or an error.
   574  //
   575  // This method may return an error for a region for which there exists a
   576  // canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
   577  // region will already be canonicalized it was obtained from a Tag that was
   578  // obtained using any of the default methods.
   579  func (r Region) TLD() (Region, error) {
   580  	// See http://en.wikipedia.org/wiki/Country_code_top-level_domain for the
   581  	// difference between ISO 3166-1 and IANA ccTLD.
   582  	if r == _GB {
   583  		r = _UK
   584  	}
   585  	if (r.typ() & ccTLD) == 0 {
   586  		return 0, errNoTLD
   587  	}
   588  	return r, nil
   589  }
   590  
   591  // Canonicalize returns the region or a possible replacement if the region is
   592  // deprecated. It will not return a replacement for deprecated regions that
   593  // are split into multiple regions.
   594  func (r Region) Canonicalize() Region {
   595  	if cr := normRegion(r); cr != 0 {
   596  		return cr
   597  	}
   598  	return r
   599  }
   600  
   601  // Variant represents a registered variant of a language as defined by BCP 47.
   602  type Variant struct {
   603  	ID  uint8
   604  	str string
   605  }
   606  
   607  // ParseVariant parses and returns a Variant. An error is returned if s is not
   608  // a valid variant.
   609  func ParseVariant(s string) (v Variant, err error) {
   610  	defer func() {
   611  		if recover() != nil {
   612  			v = Variant{}
   613  			err = ErrSyntax
   614  		}
   615  	}()
   616  
   617  	s = strings.ToLower(s)
   618  	if id, ok := variantIndex[s]; ok {
   619  		return Variant{id, s}, nil
   620  	}
   621  	return Variant{}, NewValueError([]byte(s))
   622  }
   623  
   624  // String returns the string representation of the variant.
   625  func (v Variant) String() string {
   626  	return v.str
   627  }
   628
View as plain text