summaryrefslogtreecommitdiff
path: root/vendor/golang.org/x/text/feature/plural/gen.go
blob: a0de986a24cb88db44f36eb05dabaa1b7812a4bc (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

package main

// This file generates data for the CLDR plural rules, as defined in
//    http://unicode.org/reports/tr35/tr35-numbers.html#Language_Plural_Rules
//
// We assume a slightly simplified grammar:
//
// 		condition     = and_condition ('or' and_condition)* samples
// 		and_condition = relation ('and' relation)*
// 		relation      = expr ('=' | '!=') range_list
// 		expr          = operand ('%' '10' '0'* )?
// 		operand       = 'n' | 'i' | 'f' | 't' | 'v' | 'w'
// 		range_list    = (range | value) (',' range_list)*
// 		range         = value'..'value
// 		value         = digit+
// 		digit         = 0|1|2|3|4|5|6|7|8|9
//
// 		samples       = ('@integer' sampleList)?
// 		                ('@decimal' sampleList)?
// 		sampleList    = sampleRange (',' sampleRange)* (',' ('…'|'...'))?
// 		sampleRange   = decimalValue ('~' decimalValue)?
// 		decimalValue  = value ('.' value)?
//
//		Symbol	Value
//		n	absolute value of the source number (integer and decimals).
//		i	integer digits of n.
//		v	number of visible fraction digits in n, with trailing zeros.
//		w	number of visible fraction digits in n, without trailing zeros.
//		f	visible fractional digits in n, with trailing zeros.
//		t	visible fractional digits in n, without trailing zeros.
//
// The algorithm for which the data is generated is based on the following
// observations
//
//    - the number of different sets of numbers which the plural rules use to
//      test inclusion is limited,
//    - most numbers that are tested on are < 100
//
// This allows us to define a bitmap for each number < 100 where a bit i
// indicates whether this number is included in some defined set i.
// The function matchPlural in plural.go defines how we can subsequently use
// this data to determine inclusion.
//
// There are a few languages for which this doesn't work. For one Italian and
// Azerbaijan, which both test against numbers > 100 for ordinals and Breton,
// which considers whether numbers are multiples of hundreds. The model here
// could be extended to handle Italian and Azerbaijan fairly easily (by
// considering the numbers 100, 200, 300, ..., 800, 900 in addition to the first
// 100), but for now it seems easier to just hard-code these cases.

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"log"
	"strconv"
	"strings"

	"golang.org/x/text/internal"
	"golang.org/x/text/internal/gen"
	"golang.org/x/text/language"
	"golang.org/x/text/unicode/cldr"
)

var (
	test = flag.Bool("test", false,
		"test existing tables; can be used to compare web data with package data.")
	outputFile     = flag.String("output", "tables.go", "output file")
	outputTestFile = flag.String("testoutput", "data_test.go", "output file")

	draft = flag.String("draft",
		"contributed",
		`Minimal draft requirements (approved, contributed, provisional, unconfirmed).`)
)

func main() {
	gen.Init()

	const pkg = "plural"

	gen.Repackage("gen_common.go", "common.go", pkg)
	// Read the CLDR zip file.
	r := gen.OpenCLDRCoreZip()
	defer r.Close()

	d := &cldr.Decoder{}
	d.SetDirFilter("supplemental", "main")
	d.SetSectionFilter("numbers", "plurals")
	data, err := d.DecodeZip(r)
	if err != nil {
		log.Fatalf("DecodeZip: %v", err)
	}

	w := gen.NewCodeWriter()
	defer w.WriteGoFile(*outputFile, pkg)

	gen.WriteCLDRVersion(w)

	genPlurals(w, data)

	w = gen.NewCodeWriter()
	defer w.WriteGoFile(*outputTestFile, pkg)

	genPluralsTests(w, data)
}

type pluralTest struct {
	locales string   // space-separated list of locales for this test
	form    int      // Use int instead of Form to simplify generation.
	integer []string // Entries of the form \d+ or \d+~\d+
	decimal []string // Entries of the form \f+ or \f+ +~\f+, where f is \d+\.\d+
}

func genPluralsTests(w *gen.CodeWriter, data *cldr.CLDR) {
	w.WriteType(pluralTest{})

	for _, plurals := range data.Supplemental().Plurals {
		if plurals.Type == "" {
			// The empty type is reserved for plural ranges.
			continue
		}
		tests := []pluralTest{}

		for _, pRules := range plurals.PluralRules {
			for _, rule := range pRules.PluralRule {
				test := pluralTest{
					locales: pRules.Locales,
					form:    int(countMap[rule.Count]),
				}
				scan := bufio.NewScanner(strings.NewReader(rule.Data()))
				scan.Split(splitTokens)
				var p *[]string
				for scan.Scan() {
					switch t := scan.Text(); t {
					case "@integer":
						p = &test.integer
					case "@decimal":
						p = &test.decimal
					case ",", "…":
					default:
						if p != nil {
							*p = append(*p, t)
						}
					}
				}
				tests = append(tests, test)
			}
		}
		w.WriteVar(plurals.Type+"Tests", tests)
	}
}

func genPlurals(w *gen.CodeWriter, data *cldr.CLDR) {
	for _, plurals := range data.Supplemental().Plurals {
		if plurals.Type == "" {
			continue
		}
		// Initialize setMap and inclusionMasks. They are already populated with
		// a few entries to serve as an example and to assign nice numbers to
		// common cases.

		// setMap contains sets of numbers represented by boolean arrays where
		// a true value for element i means that the number i is included.
		setMap := map[[numN]bool]int{
			// The above init func adds an entry for including all numbers.
			[numN]bool{1: true}: 1, // fix {1} to a nice value
			[numN]bool{2: true}: 2, // fix {2} to a nice value
			[numN]bool{0: true}: 3, // fix {0} to a nice value
		}

		// inclusionMasks contains bit masks for every number under numN to
		// indicate in which set the number is included. Bit 1 << x will be set
		// if it is included in set x.
		inclusionMasks := [numN]uint64{
			// Note: these entries are not complete: more bits will be set along the way.
			0: 1 << 3,
			1: 1 << 1,
			2: 1 << 2,
		}

		// Create set {0..99}. We will assign this set the identifier 0.
		var all [numN]bool
		for i := range all {
			// Mark number i as being included in the set (which has identifier 0).
			inclusionMasks[i] |= 1 << 0
			// Mark number i as included in the set.
			all[i] = true
		}
		// Register the identifier for the set.
		setMap[all] = 0

		rules := []pluralCheck{}
		index := []byte{0}
		langMap := map[int]byte{0: 0} // From compact language index to index

		for _, pRules := range plurals.PluralRules {
			// Parse the rules.
			var conds []orCondition
			for _, rule := range pRules.PluralRule {
				form := countMap[rule.Count]
				conds = parsePluralCondition(conds, rule.Data(), form)
			}
			// Encode the rules.
			for _, c := range conds {
				// If an or condition only has filters, we create an entry for
				// this filter and the set that contains all values.
				empty := true
				for _, b := range c.used {
					empty = empty && !b
				}
				if empty {
					rules = append(rules, pluralCheck{
						cat:   byte(opMod<<opShift) | byte(c.form),
						setID: 0, // all values
					})
					continue
				}
				// We have some entries with values.
				for i, set := range c.set {
					if !c.used[i] {
						continue
					}
					index, ok := setMap[set]
					if !ok {
						index = len(setMap)
						setMap[set] = index
						for i := range inclusionMasks {
							if set[i] {
								inclusionMasks[i] |= 1 << uint64(index)
							}
						}
					}
					rules = append(rules, pluralCheck{
						cat:   byte(i<<opShift | andNext),
						setID: byte(index),
					})
				}
				// Now set the last entry to the plural form the rule matches.
				rules[len(rules)-1].cat &^= formMask
				rules[len(rules)-1].cat |= byte(c.form)
			}
			// Point the relevant locales to the created entries.
			for _, loc := range strings.Split(pRules.Locales, " ") {
				if strings.TrimSpace(loc) == "" {
					continue
				}
				lang, ok := language.CompactIndex(language.MustParse(loc))
				if !ok {
					log.Printf("No compact index for locale %q", loc)
				}
				langMap[lang] = byte(len(index) - 1)
			}
			index = append(index, byte(len(rules)))
		}
		w.WriteVar(plurals.Type+"Rules", rules)
		w.WriteVar(plurals.Type+"Index", index)
		// Expand the values.
		langToIndex := make([]byte, language.NumCompactTags)
		for i := range langToIndex {
			for p := i; ; p = int(internal.Parent[p]) {
				if x, ok := langMap[p]; ok {
					langToIndex[i] = x
					break
				}
			}
		}
		w.WriteVar(plurals.Type+"LangToIndex", langToIndex)
		// Need to convert array to slice because of golang.org/issue/7651.
		// This will allow tables to be dropped when unused. This is especially
		// relevant for the ordinal data, which I suspect won't be used as much.
		w.WriteVar(plurals.Type+"InclusionMasks", inclusionMasks[:])

		if len(rules) > 0xFF {
			log.Fatalf("Too many entries for rules: %#x", len(rules))
		}
		if len(index) > 0xFF {
			log.Fatalf("Too many entries for index: %#x", len(index))
		}
		if len(setMap) > 64 { // maximum number of bits.
			log.Fatalf("Too many entries for setMap: %d", len(setMap))
		}
		w.WriteComment(
			"Slots used for %s: %X of 0xFF rules; %X of 0xFF indexes; %d of 64 sets",
			plurals.Type, len(rules), len(index), len(setMap))
		// Prevent comment from attaching to the next entry.
		fmt.Fprint(w, "\n\n")
	}
}

type orCondition struct {
	original string // for debugging

	form Form
	used [32]bool
	set  [32][numN]bool
}

func (o *orCondition) add(op opID, mod int, v []int) (ok bool) {
	ok = true
	for _, x := range v {
		if x >= maxMod {
			ok = false
			break
		}
	}
	for i := 0; i < numN; i++ {
		m := i
		if mod != 0 {
			m = i % mod
		}
		if !intIn(m, v) {
			o.set[op][i] = false
		}
	}
	if ok {
		o.used[op] = true
	}
	return ok
}

func intIn(x int, a []int) bool {
	for _, y := range a {
		if x == y {
			return true
		}
	}
	return false
}

var operandIndex = map[string]opID{
	"i": opI,
	"n": opN,
	"f": opF,
	"v": opV,
	"w": opW,
}

// parsePluralCondition parses the condition of a single pluralRule and appends
// the resulting or conditions to conds.
//
// Example rules:
//   // Category "one" in English: only allow 1 with no visible fraction
//   i = 1 and v = 0 @integer 1
//
//   // Category "few" in Czech: all numbers with visible fractions
//   v != 0   @decimal ...
//
//   // Category "zero" in Latvian: all multiples of 10 or the numbers 11-19 or
//   // numbers with a fraction 11..19 and no trailing zeros.
//   n % 10 = 0 or n % 100 = 11..19 or v = 2 and f % 100 = 11..19 @integer ...
//
// @integer and @decimal are followed by examples and are not relevant for the
// rule itself. The are used here to signal the termination of the rule.
func parsePluralCondition(conds []orCondition, s string, f Form) []orCondition {
	scan := bufio.NewScanner(strings.NewReader(s))
	scan.Split(splitTokens)
	for {
		cond := orCondition{original: s, form: f}
		// Set all numbers to be allowed for all number classes and restrict
		// from here on.
		for i := range cond.set {
			for j := range cond.set[i] {
				cond.set[i][j] = true
			}
		}
	andLoop:
		for {
			var token string
			scan.Scan() // Must exist.
			switch class := scan.Text(); class {
			case "t":
				class = "w" // equal to w for t == 0
				fallthrough
			case "n", "i", "f", "v", "w":
				op := scanToken(scan)
				opCode := operandIndex[class]
				mod := 0
				if op == "%" {
					opCode |= opMod

					switch v := scanUint(scan); v {
					case 10, 100:
						mod = v
					case 1000:
						// A more general solution would be to allow checking
						// against multiples of 100 and include entries for the
						// numbers 100..900 in the inclusion masks. At the
						// moment this would only help Azerbaijan and Italian.

						// Italian doesn't use '%', so this must be Azerbaijan.
						cond.used[opAzerbaijan00s] = true
						return append(conds, cond)

					case 1000000:
						cond.used[opBretonM] = true
						return append(conds, cond)

					default:
						log.Fatalf("Modulo value not supported %d", v)
					}
					op = scanToken(scan)
				}
				if op != "=" && op != "!=" {
					log.Fatalf("Unexpected op %q", op)
				}
				if op == "!=" {
					opCode |= opNotEqual
				}
				a := []int{}
				v := scanUint(scan)
				if class == "w" && v != 0 {
					log.Fatalf("Must compare against zero for operand type %q", class)
				}
				token = scanToken(scan)
				for {
					switch token {
					case "..":
						end := scanUint(scan)
						for ; v <= end; v++ {
							a = append(a, v)
						}
						token = scanToken(scan)
					default: // ",", "or", "and", "@..."
						a = append(a, v)
					}
					if token != "," {
						break
					}
					v = scanUint(scan)
					token = scanToken(scan)
				}
				if !cond.add(opCode, mod, a) {
					// Detected large numbers. As we ruled out Azerbaijan, this
					// must be the many rule for Italian ordinals.
					cond.set[opItalian800] = cond.set[opN]
					cond.used[opItalian800] = true
				}

			case "@integer", "@decimal": // "other" entry: tests only.
				return conds
			default:
				log.Fatalf("Unexpected operand class %q (%s)", class, s)
			}
			switch token {
			case "or":
				conds = append(conds, cond)
				break andLoop
			case "@integer", "@decimal": // examples
				// There is always an example in practice, so we always terminate here.
				if err := scan.Err(); err != nil {
					log.Fatal(err)
				}
				return append(conds, cond)
			case "and":
				// keep accumulating
			default:
				log.Fatalf("Unexpected token %q", token)
			}
		}
	}
}

func scanToken(scan *bufio.Scanner) string {
	scan.Scan()
	return scan.Text()
}

func scanUint(scan *bufio.Scanner) int {
	scan.Scan()
	val, err := strconv.ParseUint(scan.Text(), 10, 32)
	if err != nil {
		log.Fatal(err)
	}
	return int(val)
}

// splitTokens can be used with bufio.Scanner to tokenize CLDR plural rules.
func splitTokens(data []byte, atEOF bool) (advance int, token []byte, err error) {
	condTokens := [][]byte{
		[]byte(".."),
		[]byte(","),
		[]byte("!="),
		[]byte("="),
	}
	advance, token, err = bufio.ScanWords(data, atEOF)
	for _, t := range condTokens {
		if len(t) >= len(token) {
			continue
		}
		switch p := bytes.Index(token, t); {
		case p == -1:
		case p == 0:
			advance = len(t)
			token = token[:len(t)]
			return advance - len(token) + len(t), token[:len(t)], err
		case p < advance:
			// Don't split when "=" overlaps "!=".
			if t[0] == '=' && token[p-1] == '!' {
				continue
			}
			advance = p
			token = token[:p]
		}
	}
	return advance, token, err
}