language.go

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:generate go run gen.go -output tables.go
     6  
     7  package language
     8  
     9  // TODO: Remove above NOTE after:
    10  // - verifying that tables are dropped correctly (most notably matcher tables).
    11  
    12  import (
    13  	"strings"
    14  
    15  	"golang.org/x/text/internal/language"
    16  	"golang.org/x/text/internal/language/compact"
    17  )
    18  
    19  // Tag represents a BCP 47 language tag. It is used to specify an instance of a
    20  // specific language or locale. All language tag values are guaranteed to be
    21  // well-formed.
    22  type Tag compact.Tag
    23  
    24  func makeTag(t language.Tag) (tag Tag) {
    25  	return Tag(compact.Make(t))
    26  }
    27  
    28  func (t *Tag) tag() language.Tag {
    29  	return (*compact.Tag)(t).Tag()
    30  }
    31  
    32  func (t *Tag) isCompact() bool {
    33  	return (*compact.Tag)(t).IsCompact()
    34  }
    35  
    36  // TODO: improve performance.
    37  func (t *Tag) lang() language.Language { return t.tag().LangID }
    38  func (t *Tag) region() language.Region { return t.tag().RegionID }
    39  func (t *Tag) script() language.Script { return t.tag().ScriptID }
    40  
    41  // Make is a convenience wrapper for Parse that omits the error.
    42  // In case of an error, a sensible default is returned.
    43  func Make(s string) Tag {
    44  	return Default.Make(s)
    45  }
    46  
    47  // Make is a convenience wrapper for c.Parse that omits the error.
    48  // In case of an error, a sensible default is returned.
    49  func (c CanonType) Make(s string) Tag {
    50  	t, _ := c.Parse(s)
    51  	return t
    52  }
    53  
    54  // Raw returns the raw base language, script and region, without making an
    55  // attempt to infer their values.
    56  func (t Tag) Raw() (b Base, s Script, r Region) {
    57  	tt := t.tag()
    58  	return Base{tt.LangID}, Script{tt.ScriptID}, Region{tt.RegionID}
    59  }
    60  
    61  // IsRoot returns true if t is equal to language "und".
    62  func (t Tag) IsRoot() bool {
    63  	return compact.Tag(t).IsRoot()
    64  }
    65  
    66  // CanonType can be used to enable or disable various types of canonicalization.
    67  type CanonType int
    68  
    69  const (
    70  	// Replace deprecated base languages with their preferred replacements.
    71  	DeprecatedBase CanonType = 1 << iota
    72  	// Replace deprecated scripts with their preferred replacements.
    73  	DeprecatedScript
    74  	// Replace deprecated regions with their preferred replacements.
    75  	DeprecatedRegion
    76  	// Remove redundant scripts.
    77  	SuppressScript
    78  	// Normalize legacy encodings. This includes legacy languages defined in
    79  	// CLDR as well as bibliographic codes defined in ISO-639.
    80  	Legacy
    81  	// Map the dominant language of a macro language group to the macro language
    82  	// subtag. For example cmn -> zh.
    83  	Macro
    84  	// The CLDR flag should be used if full compatibility with CLDR is required.
    85  	// There are a few cases where language.Tag may differ from CLDR. To follow all
    86  	// of CLDR's suggestions, use All|CLDR.
    87  	CLDR
    88  
    89  	// Raw can be used to Compose or Parse without Canonicalization.
    90  	Raw CanonType = 0
    91  
    92  	// Replace all deprecated tags with their preferred replacements.
    93  	Deprecated = DeprecatedBase | DeprecatedScript | DeprecatedRegion
    94  
    95  	// All canonicalizations recommended by BCP 47.
    96  	BCP47 = Deprecated | SuppressScript
    97  
    98  	// All canonicalizations.
    99  	All = BCP47 | Legacy | Macro
   100  
   101  	// Default is the canonicalization used by Parse, Make and Compose. To
   102  	// preserve as much information as possible, canonicalizations that remove
   103  	// potentially valuable information are not included. The Matcher is
   104  	// designed to recognize similar tags that would be the same if
   105  	// they were canonicalized using All.
   106  	Default = Deprecated | Legacy
   107  
   108  	canonLang = DeprecatedBase | Legacy | Macro
   109  
   110  	// TODO: LikelyScript, LikelyRegion: suppress similar to ICU.
   111  )
   112  
   113  // canonicalize returns the canonicalized equivalent of the tag and
   114  // whether there was any change.
   115  func canonicalize(c CanonType, t language.Tag) (language.Tag, bool) {
   116  	if c == Raw {
   117  		return t, false
   118  	}
   119  	changed := false
   120  	if c&SuppressScript != 0 {
   121  		if t.LangID.SuppressScript() == t.ScriptID {
   122  			t.ScriptID = 0
   123  			changed = true
   124  		}
   125  	}
   126  	if c&canonLang != 0 {
   127  		for {
   128  			if l, aliasType := t.LangID.Canonicalize(); l != t.LangID {
   129  				switch aliasType {
   130  				case language.Legacy:
   131  					if c&Legacy != 0 {
   132  						if t.LangID == _sh && t.ScriptID == 0 {
   133  							t.ScriptID = _Latn
   134  						}
   135  						t.LangID = l
   136  						changed = true
   137  					}
   138  				case language.Macro:
   139  					if c&Macro != 0 {
   140  						// We deviate here from CLDR. The mapping "nb" -> "no"
   141  						// qualifies as a typical Macro language mapping.  However,
   142  						// for legacy reasons, CLDR maps "no", the macro language
   143  						// code for Norwegian, to the dominant variant "nb". This
   144  						// change is currently under consideration for CLDR as well.
   145  						// See https://unicode.org/cldr/trac/ticket/2698 and also
   146  						// https://unicode.org/cldr/trac/ticket/1790 for some of the
   147  						// practical implications. TODO: this check could be removed
   148  						// if CLDR adopts this change.
   149  						if c&CLDR == 0 || t.LangID != _nb {
   150  							changed = true
   151  							t.LangID = l
   152  						}
   153  					}
   154  				case language.Deprecated:
   155  					if c&DeprecatedBase != 0 {
   156  						if t.LangID == _mo && t.RegionID == 0 {
   157  							t.RegionID = _MD
   158  						}
   159  						t.LangID = l
   160  						changed = true
   161  						// Other canonicalization types may still apply.
   162  						continue
   163  					}
   164  				}
   165  			} else if c&Legacy != 0 && t.LangID == _no && c&CLDR != 0 {
   166  				t.LangID = _nb
   167  				changed = true
   168  			}
   169  			break
   170  		}
   171  	}
   172  	if c&DeprecatedScript != 0 {
   173  		if t.ScriptID == _Qaai {
   174  			changed = true
   175  			t.ScriptID = _Zinh
   176  		}
   177  	}
   178  	if c&DeprecatedRegion != 0 {
   179  		if r := t.RegionID.Canonicalize(); r != t.RegionID {
   180  			changed = true
   181  			t.RegionID = r
   182  		}
   183  	}
   184  	return t, changed
   185  }
   186  
   187  // Canonicalize returns the canonicalized equivalent of the tag.
   188  func (c CanonType) Canonicalize(t Tag) (Tag, error) {
   189  	// First try fast path.
   190  	if t.isCompact() {
   191  		if _, changed := canonicalize(c, compact.Tag(t).Tag()); !changed {
   192  			return t, nil
   193  		}
   194  	}
   195  	// It is unlikely that one will canonicalize a tag after matching. So do
   196  	// a slow but simple approach here.
   197  	if tag, changed := canonicalize(c, t.tag()); changed {
   198  		tag.RemakeString()
   199  		return makeTag(tag), nil
   200  	}
   201  	return t, nil
   202  
   203  }
   204  
   205  // Confidence indicates the level of certainty for a given return value.
   206  // For example, Serbian may be written in Cyrillic or Latin script.
   207  // The confidence level indicates whether a value was explicitly specified,
   208  // whether it is typically the only possible value, or whether there is
   209  // an ambiguity.
   210  type Confidence int
   211  
   212  const (
   213  	No    Confidence = iota // full confidence that there was no match
   214  	Low                     // most likely value picked out of a set of alternatives
   215  	High                    // value is generally assumed to be the correct match
   216  	Exact                   // exact match or explicitly specified value
   217  )
   218  
   219  var confName = []string{"No", "Low", "High", "Exact"}
   220  
   221  func (c Confidence) String() string {
   222  	return confName[c]
   223  }
   224  
   225  // String returns the canonical string representation of the language tag.
   226  func (t Tag) String() string {
   227  	return t.tag().String()
   228  }
   229  
   230  // MarshalText implements encoding.TextMarshaler.
   231  func (t Tag) MarshalText() (text []byte, err error) {
   232  	return t.tag().MarshalText()
   233  }
   234  
   235  // UnmarshalText implements encoding.TextUnmarshaler.
   236  func (t *Tag) UnmarshalText(text []byte) error {
   237  	var tag language.Tag
   238  	err := tag.UnmarshalText(text)
   239  	*t = makeTag(tag)
   240  	return err
   241  }
   242  
   243  // Base returns the base language of the language tag. If the base language is
   244  // unspecified, an attempt will be made to infer it from the context.
   245  // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
   246  func (t Tag) Base() (Base, Confidence) {
   247  	if b := t.lang(); b != 0 {
   248  		return Base{b}, Exact
   249  	}
   250  	tt := t.tag()
   251  	c := High
   252  	if tt.ScriptID == 0 && !tt.RegionID.IsCountry() {
   253  		c = Low
   254  	}
   255  	if tag, err := tt.Maximize(); err == nil && tag.LangID != 0 {
   256  		return Base{tag.LangID}, c
   257  	}
   258  	return Base{0}, No
   259  }
   260  
   261  // Script infers the script for the language tag. If it was not explicitly given, it will infer
   262  // a most likely candidate.
   263  // If more than one script is commonly used for a language, the most likely one
   264  // is returned with a low confidence indication. For example, it returns (Cyrl, Low)
   265  // for Serbian.
   266  // If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined)
   267  // as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks
   268  // common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts.
   269  // See https://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for
   270  // unknown value in CLDR.  (Zzzz, Exact) is returned if Zzzz was explicitly specified.
   271  // Note that an inferred script is never guaranteed to be the correct one. Latin is
   272  // almost exclusively used for Afrikaans, but Arabic has been used for some texts
   273  // in the past.  Also, the script that is commonly used may change over time.
   274  // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
   275  func (t Tag) Script() (Script, Confidence) {
   276  	if scr := t.script(); scr != 0 {
   277  		return Script{scr}, Exact
   278  	}
   279  	tt := t.tag()
   280  	sc, c := language.Script(_Zzzz), No
   281  	if scr := tt.LangID.SuppressScript(); scr != 0 {
   282  		// Note: it is not always the case that a language with a suppress
   283  		// script value is only written in one script (e.g. kk, ms, pa).
   284  		if tt.RegionID == 0 {
   285  			return Script{scr}, High
   286  		}
   287  		sc, c = scr, High
   288  	}
   289  	if tag, err := tt.Maximize(); err == nil {
   290  		if tag.ScriptID != sc {
   291  			sc, c = tag.ScriptID, Low
   292  		}
   293  	} else {
   294  		tt, _ = canonicalize(Deprecated|Macro, tt)
   295  		if tag, err := tt.Maximize(); err == nil && tag.ScriptID != sc {
   296  			sc, c = tag.ScriptID, Low
   297  		}
   298  	}
   299  	return Script{sc}, c
   300  }
   301  
   302  // Region returns the region for the language tag. If it was not explicitly given, it will
   303  // infer a most likely candidate from the context.
   304  // It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
   305  func (t Tag) Region() (Region, Confidence) {
   306  	if r := t.region(); r != 0 {
   307  		return Region{r}, Exact
   308  	}
   309  	tt := t.tag()
   310  	if tt, err := tt.Maximize(); err == nil {
   311  		return Region{tt.RegionID}, Low // TODO: differentiate between high and low.
   312  	}
   313  	tt, _ = canonicalize(Deprecated|Macro, tt)
   314  	if tag, err := tt.Maximize(); err == nil {
   315  		return Region{tag.RegionID}, Low
   316  	}
   317  	return Region{_ZZ}, No // TODO: return world instead of undetermined?
   318  }
   319  
   320  // Variants returns the variants specified explicitly for this language tag.
   321  // or nil if no variant was specified.
   322  func (t Tag) Variants() []Variant {
   323  	if !compact.Tag(t).MayHaveVariants() {
   324  		return nil
   325  	}
   326  	v := []Variant{}
   327  	x, str := "", t.tag().Variants()
   328  	for str != "" {
   329  		x, str = nextToken(str)
   330  		v = append(v, Variant{x})
   331  	}
   332  	return v
   333  }
   334  
   335  // Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
   336  // specific language are substituted with fields from the parent language.
   337  // The parent for a language may change for newer versions of CLDR.
   338  //
   339  // Parent returns a tag for a less specific language that is mutually
   340  // intelligible or Und if there is no such language. This may not be the same as
   341  // simply stripping the last BCP 47 subtag. For instance, the parent of "zh-TW"
   342  // is "zh-Hant", and the parent of "zh-Hant" is "und".
   343  func (t Tag) Parent() Tag {
   344  	return Tag(compact.Tag(t).Parent())
   345  }
   346  
   347  // nextToken returns token t and the rest of the string.
   348  func nextToken(s string) (t, tail string) {
   349  	p := strings.Index(s[1:], "-")
   350  	if p == -1 {
   351  		return s[1:], ""
   352  	}
   353  	p++
   354  	return s[1:p], s[p:]
   355  }
   356  
   357  // Extension is a single BCP 47 extension.
   358  type Extension struct {
   359  	s string
   360  }
   361  
   362  // String returns the string representation of the extension, including the
   363  // type tag.
   364  func (e Extension) String() string {
   365  	return e.s
   366  }
   367  
   368  // ParseExtension parses s as an extension and returns it on success.
   369  func ParseExtension(s string) (e Extension, err error) {
   370  	ext, err := language.ParseExtension(s)
   371  	return Extension{ext}, err
   372  }
   373  
   374  // Type returns the one-byte extension type of e. It returns 0 for the zero
   375  // exception.
   376  func (e Extension) Type() byte {
   377  	if e.s == "" {
   378  		return 0
   379  	}
   380  	return e.s[0]
   381  }
   382  
   383  // Tokens returns the list of tokens of e.
   384  func (e Extension) Tokens() []string {
   385  	return strings.Split(e.s, "-")
   386  }
   387  
   388  // Extension returns the extension of type x for tag t. It will return
   389  // false for ok if t does not have the requested extension. The returned
   390  // extension will be invalid in this case.
   391  func (t Tag) Extension(x byte) (ext Extension, ok bool) {
   392  	if !compact.Tag(t).MayHaveExtensions() {
   393  		return Extension{}, false
   394  	}
   395  	e, ok := t.tag().Extension(x)
   396  	return Extension{e}, ok
   397  }
   398  
   399  // Extensions returns all extensions of t.
   400  func (t Tag) Extensions() []Extension {
   401  	if !compact.Tag(t).MayHaveExtensions() {
   402  		return nil
   403  	}
   404  	e := []Extension{}
   405  	for _, ext := range t.tag().Extensions() {
   406  		e = append(e, Extension{ext})
   407  	}
   408  	return e
   409  }
   410  
   411  // TypeForKey returns the type associated with the given key, where key and type
   412  // are of the allowed values defined for the Unicode locale extension ('u') in
   413  // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
   414  // TypeForKey will traverse the inheritance chain to get the correct value.
   415  //
   416  // If there are multiple types associated with a key, only the first will be
   417  // returned. If there is no type associated with a key, it returns the empty
   418  // string.
   419  func (t Tag) TypeForKey(key string) string {
   420  	if !compact.Tag(t).MayHaveExtensions() {
   421  		if key != "rg" && key != "va" {
   422  			return ""
   423  		}
   424  	}
   425  	return t.tag().TypeForKey(key)
   426  }
   427  
   428  // SetTypeForKey returns a new Tag with the key set to type, where key and type
   429  // are of the allowed values defined for the Unicode locale extension ('u') in
   430  // https://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
   431  // An empty value removes an existing pair with the same key.
   432  func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
   433  	tt, err := t.tag().SetTypeForKey(key, value)
   434  	return makeTag(tt), err
   435  }
   436  
   437  // NumCompactTags is the number of compact tags. The maximum tag is
   438  // NumCompactTags-1.
   439  const NumCompactTags = compact.NumCompactTags
   440  
   441  // CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags
   442  // for which data exists in the text repository.The index will change over time
   443  // and should not be stored in persistent storage. If t does not match a compact
   444  // index, exact will be false and the compact index will be returned for the
   445  // first match after repeatedly taking the Parent of t.
   446  func CompactIndex(t Tag) (index int, exact bool) {
   447  	id, exact := compact.LanguageID(compact.Tag(t))
   448  	return int(id), exact
   449  }
   450  
   451  var root = language.Tag{}
   452  
   453  // Base is an ISO 639 language code, used for encoding the base language
   454  // of a language tag.
   455  type Base struct {
   456  	langID language.Language
   457  }
   458  
   459  // ParseBase parses a 2- or 3-letter ISO 639 code.
   460  // It returns a ValueError if s is a well-formed but unknown language identifier
   461  // or another error if another error occurred.
   462  func ParseBase(s string) (Base, error) {
   463  	l, err := language.ParseBase(s)
   464  	return Base{l}, err
   465  }
   466  
   467  // String returns the BCP 47 representation of the base language.
   468  func (b Base) String() string {
   469  	return b.langID.String()
   470  }
   471  
   472  // ISO3 returns the ISO 639-3 language code.
   473  func (b Base) ISO3() string {
   474  	return b.langID.ISO3()
   475  }
   476  
   477  // IsPrivateUse reports whether this language code is reserved for private use.
   478  func (b Base) IsPrivateUse() bool {
   479  	return b.langID.IsPrivateUse()
   480  }
   481  
   482  // Script is a 4-letter ISO 15924 code for representing scripts.
   483  // It is idiomatically represented in title case.
   484  type Script struct {
   485  	scriptID language.Script
   486  }
   487  
   488  // ParseScript parses a 4-letter ISO 15924 code.
   489  // It returns a ValueError if s is a well-formed but unknown script identifier
   490  // or another error if another error occurred.
   491  func ParseScript(s string) (Script, error) {
   492  	sc, err := language.ParseScript(s)
   493  	return Script{sc}, err
   494  }
   495  
   496  // String returns the script code in title case.
   497  // It returns "Zzzz" for an unspecified script.
   498  func (s Script) String() string {
   499  	return s.scriptID.String()
   500  }
   501  
   502  // IsPrivateUse reports whether this script code is reserved for private use.
   503  func (s Script) IsPrivateUse() bool {
   504  	return s.scriptID.IsPrivateUse()
   505  }
   506  
   507  // Region is an ISO 3166-1 or UN M.49 code for representing countries and regions.
   508  type Region struct {
   509  	regionID language.Region
   510  }
   511  
   512  // EncodeM49 returns the Region for the given UN M.49 code.
   513  // It returns an error if r is not a valid code.
   514  func EncodeM49(r int) (Region, error) {
   515  	rid, err := language.EncodeM49(r)
   516  	return Region{rid}, err
   517  }
   518  
   519  // ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
   520  // It returns a ValueError if s is a well-formed but unknown region identifier
   521  // or another error if another error occurred.
   522  func ParseRegion(s string) (Region, error) {
   523  	r, err := language.ParseRegion(s)
   524  	return Region{r}, err
   525  }
   526  
   527  // String returns the BCP 47 representation for the region.
   528  // It returns "ZZ" for an unspecified region.
   529  func (r Region) String() string {
   530  	return r.regionID.String()
   531  }
   532  
   533  // ISO3 returns the 3-letter ISO code of r.
   534  // Note that not all regions have a 3-letter ISO code.
   535  // In such cases this method returns "ZZZ".
   536  func (r Region) ISO3() string {
   537  	return r.regionID.ISO3()
   538  }
   539  
   540  // M49 returns the UN M.49 encoding of r, or 0 if this encoding
   541  // is not defined for r.
   542  func (r Region) M49() int {
   543  	return r.regionID.M49()
   544  }
   545  
   546  // IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This
   547  // may include private-use tags that are assigned by CLDR and used in this
   548  // implementation. So IsPrivateUse and IsCountry can be simultaneously true.
   549  func (r Region) IsPrivateUse() bool {
   550  	return r.regionID.IsPrivateUse()
   551  }
   552  
   553  // IsCountry returns whether this region is a country or autonomous area. This
   554  // includes non-standard definitions from CLDR.
   555  func (r Region) IsCountry() bool {
   556  	return r.regionID.IsCountry()
   557  }
   558  
   559  // IsGroup returns whether this region defines a collection of regions. This
   560  // includes non-standard definitions from CLDR.
   561  func (r Region) IsGroup() bool {
   562  	return r.regionID.IsGroup()
   563  }
   564  
   565  // Contains returns whether Region c is contained by Region r. It returns true
   566  // if c == r.
   567  func (r Region) Contains(c Region) bool {
   568  	return r.regionID.Contains(c.regionID)
   569  }
   570  
   571  // TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
   572  // In all other cases it returns either the region itself or an error.
   573  //
   574  // This method may return an error for a region for which there exists a
   575  // canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
   576  // region will already be canonicalized it was obtained from a Tag that was
   577  // obtained using any of the default methods.
   578  func (r Region) TLD() (Region, error) {
   579  	tld, err := r.regionID.TLD()
   580  	return Region{tld}, err
   581  }
   582  
   583  // Canonicalize returns the region or a possible replacement if the region is
   584  // deprecated. It will not return a replacement for deprecated regions that
   585  // are split into multiple regions.
   586  func (r Region) Canonicalize() Region {
   587  	return Region{r.regionID.Canonicalize()}
   588  }
   589  
   590  // Variant represents a registered variant of a language as defined by BCP 47.
   591  type Variant struct {
   592  	variant string
   593  }
   594  
   595  // ParseVariant parses and returns a Variant. An error is returned if s is not
   596  // a valid variant.
   597  func ParseVariant(s string) (Variant, error) {
   598  	v, err := language.ParseVariant(s)
   599  	return Variant{v.String()}, err
   600  }
   601  
   602  // String returns the string representation of the variant.
   603  func (v Variant) String() string {
   604  	return v.variant
   605  }
   606
View as plain text