// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package norm import ( "strings" "testing" ) var iterBufSizes = []int{ MaxSegmentSize, 1.5 * MaxSegmentSize, 2 * MaxSegmentSize, 3 * MaxSegmentSize, 100 * MaxSegmentSize, } func doIterNorm(f Form, buf []byte, s string) []byte { acc := []byte{} i := Iter{} i.SetInputString(f, s) for !i.Done() { n := i.Next(buf) acc = append(acc, buf[:n]...) } return acc } func runIterTests(t *testing.T, name string, f Form, tests []AppendTest, norm bool) { for i, test := range tests { in := test.left + test.right gold := test.out if norm { gold = string(f.AppendString(nil, test.out)) } for _, sz := range iterBufSizes { buf := make([]byte, sz) out := string(doIterNorm(f, buf, in)) if len(out) != len(gold) { const msg = "%s:%d:%d: length is %d; want %d" t.Errorf(msg, name, i, sz, len(out), len(gold)) } if out != gold { // Find first rune that differs and show context. ir := []rune(out) ig := []rune(gold) for j := 0; j < len(ir) && j < len(ig); j++ { if ir[j] == ig[j] { continue } if j -= 3; j < 0 { j = 0 } for e := j + 7; j < e && j < len(ir) && j < len(ig); j++ { const msg = "%s:%d:%d: runeAt(%d) = %U; want %U" t.Errorf(msg, name, i, sz, j, ir[j], ig[j]) } break } } } } } func rep(r rune, n int) string { return strings.Repeat(string(r), n) } var iterTests = []AppendTest{ {"", ascii, ascii}, {"", txt_all, txt_all}, {"", "a" + rep(0x0300, MaxSegmentSize/2), "a" + rep(0x0300, MaxSegmentSize/2)}, } var iterTestsD = []AppendTest{ { // segment overflow on unchanged character "", "a" + rep(0x0300, MaxSegmentSize/2) + "\u0316", "a" + rep(0x0300, MaxSegmentSize/2-1) + "\u0316\u0300", }, { // segment overflow on unchanged character + start value "", "a" + rep(0x0300, MaxSegmentSize/2+maxCombiningChars+4) + "\u0316", "a" + rep(0x0300, MaxSegmentSize/2+maxCombiningChars) + "\u0316" + rep(0x300, 4), }, { // segment overflow on decomposition "", "a" + rep(0x0300, MaxSegmentSize/2-1) + "\u0340", "a" + rep(0x0300, MaxSegmentSize/2), }, { // segment overflow on decomposition + start value "", "a" + rep(0x0300, MaxSegmentSize/2-1) + "\u0340" + rep(0x300, maxCombiningChars+4) + "\u0320", "a" + rep(0x0300, MaxSegmentSize/2-1) + rep(0x300, maxCombiningChars+1) + "\u0320" + rep(0x300, 4), }, { // start value after ASCII overflow "", rep('a', MaxSegmentSize) + rep(0x300, maxCombiningChars+2) + "\u0320", rep('a', MaxSegmentSize) + rep(0x300, maxCombiningChars) + "\u0320\u0300\u0300", }, { // start value after Hangul overflow "", rep(0xAC00, MaxSegmentSize/6) + rep(0x300, maxCombiningChars+2) + "\u0320", strings.Repeat("\u1100\u1161", MaxSegmentSize/6) + rep(0x300, maxCombiningChars-1) + "\u0320" + rep(0x300, 3), }, { // start value after cc=0 "", "您您" + rep(0x300, maxCombiningChars+4) + "\u0320", "您您" + rep(0x300, maxCombiningChars) + "\u0320" + rep(0x300, 4), }, { // start value after normalization "", "\u0300\u0320a" + rep(0x300, maxCombiningChars+4) + "\u0320", "\u0320\u0300a" + rep(0x300, maxCombiningChars) + "\u0320" + rep(0x300, 4), }, } var iterTestsC = []AppendTest{ { // ordering of non-composing combining characters "", "\u0305\u0316", "\u0316\u0305", }, { // segment overflow "", "a" + rep(0x0305, MaxSegmentSize/2+4) + "\u0316", "a" + rep(0x0305, MaxSegmentSize/2-1) + "\u0316" + rep(0x305, 5), }, } func TestIterNextD(t *testing.T) { runIterTests(t, "IterNextD1", NFKD, appendTests, true) runIterTests(t, "IterNextD2", NFKD, iterTests, true) runIterTests(t, "IterNextD3", NFKD, iterTestsD, false) } func TestIterNextC(t *testing.T) { runIterTests(t, "IterNextC1", NFKC, appendTests, true) runIterTests(t, "IterNextC2", NFKC, iterTests, true) runIterTests(t, "IterNextC3", NFKC, iterTestsC, false) } type SegmentTest struct { in string out []string } var segmentTests = []SegmentTest{ {rep('a', MaxSegmentSize), []string{rep('a', MaxSegmentSize), ""}}, {rep('a', MaxSegmentSize+2), []string{rep('a', MaxSegmentSize-1), "aaa", ""}}, {rep('a', MaxSegmentSize) + "\u0300aa", []string{rep('a', MaxSegmentSize-1), "a\u0300", "aa", ""}}, } // Note that, by design, segmentation is equal for composing and decomposing forms. func TestIterSegmentation(t *testing.T) { segmentTest(t, "SegmentTestD", NFD, segmentTests) segmentTest(t, "SegmentTestC", NFC, segmentTests) } func segmentTest(t *testing.T, name string, f Form, tests []SegmentTest) { iter := Iter{} for i, tt := range segmentTests { buf := make([]byte, MaxSegmentSize) iter.SetInputString(f, tt.in) for j, seg := range tt.out { if seg == "" { if !iter.Done() { n := iter.Next(buf) res := string(buf[:n]) t.Errorf(`%s:%d:%d: expected Done()==true, found segment "%s"`, name, i, j, res) } continue } if iter.Done() { t.Errorf("%s:%d:%d: Done()==true, want false", name, i, j) } n := iter.Next(buf) seg = f.String(seg) if res := string(buf[:n]); res != seg { t.Errorf(`%s:%d:%d" segment was "%s" (%d); want "%s" (%d)`, name, i, j, res, len(res), seg, len(seg)) } } } }