From 04f99b387021a8ce32a8795360cba9beaf986a81 Mon Sep 17 00:00:00 2001 From: Ondřej Surý Date: Tue, 13 Sep 2011 12:00:31 +0200 Subject: Imported Upstream version 2011.09.07 --- src/Make.inc | 6 +- src/Make.pkg | 4 +- src/cmd/5a/lex.c | 16 +- src/cmd/5c/gc.h | 2 +- src/cmd/5c/swt.c | 12 +- src/cmd/5g/cgen.c | 24 +- src/cmd/5g/cgen64.c | 12 +- src/cmd/5g/galign.c | 2 + src/cmd/5g/gg.h | 17 +- src/cmd/5g/ggen.c | 4 +- src/cmd/5g/gobj.c | 61 +- src/cmd/5g/gsubr.c | 42 +- src/cmd/5g/list.c | 2 + src/cmd/5g/peep.c | 10 +- src/cmd/5g/reg.c | 31 +- src/cmd/5l/softfloat.c | 1 - src/cmd/6a/lex.c | 20 +- src/cmd/6c/swt.c | 12 +- src/cmd/6g/cgen.c | 21 +- src/cmd/6g/galign.c | 2 + src/cmd/6g/gg.h | 16 +- src/cmd/6g/ggen.c | 25 +- src/cmd/6g/gobj.c | 63 +- src/cmd/6g/gsubr.c | 42 +- src/cmd/6g/list.c | 4 +- src/cmd/6g/peep.c | 3 +- src/cmd/6g/reg.c | 40 +- src/cmd/6l/6.out.h | 3 +- src/cmd/6l/optab.c | 3 +- src/cmd/6l/span.c | 29 +- src/cmd/8a/a.y | 7 + src/cmd/8a/lex.c | 19 +- src/cmd/8c/swt.c | 12 +- src/cmd/8g/cgen.c | 20 +- src/cmd/8g/cgen64.c | 2 + src/cmd/8g/galign.c | 2 + src/cmd/8g/gg.h | 20 +- src/cmd/8g/ggen.c | 22 +- src/cmd/8g/gobj.c | 64 +- src/cmd/8g/gsubr.c | 50 +- src/cmd/8g/list.c | 4 +- src/cmd/8g/opt.h | 2 + src/cmd/8g/peep.c | 3 +- src/cmd/8g/reg.c | 34 +- src/cmd/8l/8.out.h | 3 +- src/cmd/8l/asm.c | 9 +- src/cmd/8l/doc.go | 2 + src/cmd/8l/l.h | 1 + src/cmd/8l/obj.c | 3 + src/cmd/8l/optab.c | 6 +- src/cmd/8l/pass.c | 7 +- src/cmd/8l/span.c | 39 +- src/cmd/cgo/doc.go | 3 + src/cmd/cov/Makefile | 1 + src/cmd/cov/main.c | 3 + src/cmd/cov/tree.c | 5 +- src/cmd/gc/Makefile | 1 + src/cmd/gc/align.c | 2 + src/cmd/gc/bits.c | 8 +- src/cmd/gc/closure.c | 5 +- src/cmd/gc/const.c | 5 +- src/cmd/gc/cplx.c | 2 + src/cmd/gc/dcl.c | 13 +- src/cmd/gc/doc.go | 5 +- src/cmd/gc/esc.c | 723 +++++++++++ src/cmd/gc/export.c | 2 + src/cmd/gc/gen.c | 169 ++- src/cmd/gc/go.h | 131 +- src/cmd/gc/go.y | 14 +- src/cmd/gc/init.c | 2 + src/cmd/gc/lex.c | 71 +- src/cmd/gc/md5.c | 2 + src/cmd/gc/mparith1.c | 2 + src/cmd/gc/mparith2.c | 6 + src/cmd/gc/mparith3.c | 2 + src/cmd/gc/obj.c | 25 +- src/cmd/gc/pgen.c | 15 +- src/cmd/gc/print.c | 6 + src/cmd/gc/range.c | 34 +- src/cmd/gc/reflect.c | 35 +- src/cmd/gc/select.c | 14 +- src/cmd/gc/sinit.c | 432 ++++++- src/cmd/gc/subr.c | 85 +- src/cmd/gc/swt.c | 18 +- src/cmd/gc/typecheck.c | 109 +- src/cmd/gc/unsafe.c | 2 + src/cmd/gc/walk.c | 98 +- src/cmd/godefs/main.c | 5 +- src/cmd/godoc/Makefile | 1 + src/cmd/godoc/appconfig.go | 15 +- src/cmd/godoc/appinit.go | 25 +- src/cmd/godoc/codewalk.go | 7 +- src/cmd/godoc/dirtrees.go | 2 +- src/cmd/godoc/doc.go | 11 + src/cmd/godoc/godoc.go | 94 +- src/cmd/godoc/index.go | 300 +++-- src/cmd/godoc/main.go | 46 +- src/cmd/godoc/parser.go | 18 +- src/cmd/godoc/snippet.go | 0 src/cmd/godoc/throttle.go | 88 ++ src/cmd/godoc/zip.go | 6 +- src/cmd/gofix/Makefile | 1 + src/cmd/gofix/netudpgroup.go | 57 + src/cmd/gofix/netudpgroup_test.go | 33 + src/cmd/gofix/osopen.go | 1 + src/cmd/gofix/osopen_test.go | 23 + src/cmd/gofix/url.go | 14 +- src/cmd/gofix/url_test.go | 12 + src/cmd/goinstall/Makefile | 6 + src/cmd/goinstall/doc.go | 7 +- src/cmd/goinstall/download.go | 223 ++-- src/cmd/goinstall/main.go | 31 +- src/cmd/goinstall/tag_test.go | 73 ++ src/cmd/gopack/ar.c | 2 + src/cmd/gotest/gotest.go | 15 +- src/cmd/gotry/gotry | 2 +- src/cmd/gotype/testdata/test1.go | 17 + src/cmd/hgpatch/main.go | 13 +- src/cmd/ld/data.c | 6 +- src/cmd/ld/elf.c | 13 + src/cmd/ld/lib.c | 26 +- src/cmd/ld/pe.c | 89 +- src/cmd/ld/symtab.c | 5 +- src/cmd/prof/Makefile | 1 + src/cmd/prof/gopprof | 6 +- src/cmd/prof/main.c | 4 + src/lib9/exitcode.c | 1 + src/libmach/executable.c | 191 ++- src/libmach/fakeobj.c | 28 +- src/libmach/linux.c | 4 + src/libmach/map.c | 2 + src/pkg/Makefile | 25 +- src/pkg/archive/tar/testdata/writer.tar | Bin 3072 -> 3584 bytes src/pkg/archive/tar/writer.go | 2 +- src/pkg/archive/tar/writer_test.go | 19 + src/pkg/big/int.go | 0 src/pkg/big/int_test.go | 3 + src/pkg/big/nat.go | 2 +- src/pkg/big/nat_test.go | 0 src/pkg/bufio/bufio.go | 24 +- src/pkg/bufio/bufio_test.go | 70 +- src/pkg/bytes/buffer.go | 9 +- src/pkg/container/heap/heap_test.go | 32 +- src/pkg/crypto/rand/rand_windows.go | 0 src/pkg/crypto/x509/verify_test.go | 10 +- src/pkg/exp/norm/Makefile | 1 + src/pkg/exp/norm/composition.go | 67 +- src/pkg/exp/norm/forminfo.go | 11 +- src/pkg/exp/norm/maketables.go | 8 +- src/pkg/exp/norm/normalize.go | 374 +++++- src/pkg/exp/norm/normalize_test.go | 624 +++++++++ src/pkg/exp/norm/readwriter.go | 121 ++ src/pkg/exp/norm/readwriter_test.go | 69 + src/pkg/exp/norm/tables.go | 1133 ++++++++-------- src/pkg/exp/norm/trie.go | 20 - src/pkg/exp/norm/trie_test.go | 6 +- src/pkg/exp/regexp/exec.go | 29 +- src/pkg/exp/regexp/exec_test.go | 271 ++++ src/pkg/exp/regexp/find_test.go | 17 + src/pkg/exp/regexp/re2.txt.gz | Bin 0 -> 764518 bytes src/pkg/exp/regexp/regexp.go | 72 +- src/pkg/exp/regexp/syntax/compile.go | 22 +- src/pkg/exp/regexp/syntax/parse.go | 26 +- src/pkg/exp/regexp/syntax/parse_test.go | 12 + src/pkg/exp/regexp/syntax/prog.go | 58 +- src/pkg/exp/regexp/syntax/prog_test.go | 10 + src/pkg/exp/regexp/syntax/regexp.go | 14 + src/pkg/exp/template/html/Makefile | 4 +- src/pkg/exp/template/html/context.go | 125 +- src/pkg/exp/template/html/escape.go | 658 +++++++++- src/pkg/exp/template/html/escape_test.go | 638 ++++++++- src/pkg/exp/template/html/js.go | 290 +++++ src/pkg/exp/template/html/js_test.go | 356 +++++ src/pkg/exp/wingui/gui.go | 11 +- src/pkg/exp/wingui/winapi.go | 45 +- src/pkg/exp/wingui/zwinapi.go | 38 +- src/pkg/flag/flag.go | 19 +- src/pkg/flag/flag_test.go | 6 + src/pkg/fmt/fmt_test.go | 28 + src/pkg/go/ast/filter.go | 301 ++--- src/pkg/go/build/build_test.go | 66 +- src/pkg/go/build/dir.go | 149 ++- src/pkg/go/build/pkgtest/pkgtest.go | 6 +- src/pkg/go/build/pkgtest/sqrt_386_test.go | 1 + src/pkg/go/build/pkgtest/sqrt_amd64_test.go | 1 + src/pkg/go/build/pkgtest/sqrt_arm_test.go | 1 + src/pkg/go/build/pkgtest/sqrt_test.go | 5 + src/pkg/go/build/pkgtest/xsqrt_test.go | 5 + src/pkg/go/build/syslist_test.go | 2 +- src/pkg/go/parser/parser.go | 88 +- src/pkg/go/parser/parser_test.go | 8 +- src/pkg/go/printer/nodes.go | 16 +- src/pkg/go/printer/testdata/declarations.golden | 111 +- src/pkg/go/printer/testdata/declarations.input | 76 ++ src/pkg/gob/debug.go | 1 + src/pkg/http/Makefile | 1 + src/pkg/http/cgi/host.go | 14 +- src/pkg/http/fcgi/fcgi_test.go | 6 +- src/pkg/http/filetransport.go | 124 ++ src/pkg/http/filetransport_test.go | 63 + src/pkg/http/request.go | 79 +- src/pkg/http/request_test.go | 51 - src/pkg/http/serve_test.go | 58 +- src/pkg/http/server.go | 83 +- src/pkg/http/spdy/read.go | 6 +- src/pkg/http/transfer.go | 11 + src/pkg/http/transport.go | 6 +- src/pkg/image/image.go | 14 + src/pkg/image/png/reader.go | 10 + src/pkg/image/png/reader_test.go | 31 +- src/pkg/image/png/testdata/invalid-crc32.png | Bin 0 -> 1289 bytes src/pkg/image/png/testdata/invalid-noend.png | Bin 0 -> 1277 bytes src/pkg/image/png/testdata/invalid-trunc.png | Bin 0 -> 1288 bytes src/pkg/image/png/testdata/invalid-zlib.png | Bin 0 -> 1289 bytes src/pkg/image/png/writer.go | 24 +- src/pkg/image/png/writer_test.go | 4 +- src/pkg/image/tiff/reader.go | 53 +- src/pkg/image/tiff/reader_test.go | 25 + .../tiff/testdata/video-001-uncompressed.tiff | Bin 0 -> 46674 bytes src/pkg/io/io.go | 24 + src/pkg/io/io_test.go | 27 + src/pkg/json/Makefile | 1 + src/pkg/json/decode.go | 22 +- src/pkg/json/decode_test.go | 13 +- src/pkg/json/encode.go | 59 +- src/pkg/json/encode_test.go | 38 + src/pkg/json/indent.go | 1 + src/pkg/json/scanner_test.go | 25 + src/pkg/json/tags.go | 44 + src/pkg/json/tags_test.go | 28 + src/pkg/math/all_test.go | 34 + src/pkg/math/atan2_decl.go | 0 src/pkg/math/pow10.go | 6 + src/pkg/mime/grammar.go | 9 + src/pkg/mime/mediatype.go | 77 +- src/pkg/mime/mediatype_test.go | 19 +- src/pkg/mime/mime_test.go | 27 - src/pkg/mime/multipart/multipart.go | 5 +- src/pkg/mime/type.go | 35 +- src/pkg/mime/type_test.go | 35 + src/pkg/net/dnsclient_unix.go | 2 +- src/pkg/net/interface.go | 57 + src/pkg/net/interface_test.go | 46 + src/pkg/net/ip.go | 252 ++-- src/pkg/net/ip_test.go | 240 +++- src/pkg/net/iprawsock_posix.go | 4 +- src/pkg/net/multicast_test.go | 116 +- src/pkg/net/parse.go | 43 + src/pkg/net/server_test.go | 4 +- src/pkg/net/tcpsock_posix.go | 7 +- src/pkg/net/udpsock_plan9.go | 11 +- src/pkg/net/udpsock_posix.go | 98 +- src/pkg/os/error_plan9.go | 1 + src/pkg/os/exec_unix.go | 3 +- src/pkg/os/file_posix.go | 33 - src/pkg/os/file_unix.go | 33 + src/pkg/os/file_windows.go | 40 +- src/pkg/os/os_test.go | 6 + src/pkg/os/stat_windows.go | 68 +- src/pkg/path/filepath/path_test.go | 2 +- src/pkg/reflect/all_test.go | 31 +- src/pkg/reflect/type.go | 4 +- src/pkg/reflect/value.go | 56 + src/pkg/rpc/server_test.go | 91 +- src/pkg/runtime/386/asm.s | 8 +- src/pkg/runtime/Makefile | 3 +- src/pkg/runtime/amd64/asm.s | 18 +- src/pkg/runtime/amd64/traceback.c | 53 +- src/pkg/runtime/arm/traceback.c | 51 +- src/pkg/runtime/cgo/amd64.S | 19 + src/pkg/runtime/cgo/util.c | 2 +- src/pkg/runtime/cgocall.c | 46 +- src/pkg/runtime/chan.c | 10 +- src/pkg/runtime/chan_test.go | 51 + src/pkg/runtime/goc2c.c | 28 +- src/pkg/runtime/linux/386/rt0.s | 10 +- src/pkg/runtime/linux/386/signal.c | 24 + src/pkg/runtime/linux/386/sys.s | 46 +- src/pkg/runtime/linux/mem.c | 17 +- src/pkg/runtime/malloc.goc | 11 +- src/pkg/runtime/mgc0.c | 1 + src/pkg/runtime/mkasmh.sh | 1 + src/pkg/runtime/openbsd/386/defs.h | 140 ++ src/pkg/runtime/openbsd/386/rt0.s | 6 + src/pkg/runtime/openbsd/386/signal.c | 189 +++ src/pkg/runtime/openbsd/386/sys.s | 254 ++++ src/pkg/runtime/openbsd/amd64/sys.s | 10 +- src/pkg/runtime/openbsd/defs.c | 4 +- src/pkg/runtime/openbsd/mem.c | 20 +- src/pkg/runtime/openbsd/thread.c | 16 +- src/pkg/runtime/proc.c | 64 +- src/pkg/runtime/runtime.c | 45 +- src/pkg/runtime/runtime.h | 24 +- src/pkg/runtime/sema.goc | 1 + src/pkg/runtime/string.goc | 2 + src/pkg/runtime/symtab.c | 37 + src/pkg/runtime/syscall_windows_test.go | 99 ++ src/pkg/runtime/windows/386/callback.c | 107 ++ src/pkg/runtime/windows/386/sys.s | 47 +- src/pkg/runtime/windows/amd64/callback.c | 104 ++ src/pkg/runtime/windows/amd64/defs.h | 58 +- src/pkg/runtime/windows/amd64/signal.c | 90 +- src/pkg/runtime/windows/amd64/sys.s | 262 +++- src/pkg/runtime/windows/defs.c | 5 + src/pkg/runtime/windows/os.h | 2 +- src/pkg/runtime/windows/syscall.goc | 4 + src/pkg/runtime/windows/thread.c | 137 +- src/pkg/sort/export_test.go | 9 + src/pkg/sort/sort.go | 61 +- src/pkg/sort/sort_test.go | 68 +- src/pkg/strconv/atof.go | 58 +- src/pkg/strconv/atof_test.go | 1 + src/pkg/strconv/decimal.go | 11 +- src/pkg/strconv/decimal_test.go | 8 +- src/pkg/strconv/ftoa.go | 9 +- src/pkg/sync/atomic/asm_386.s | 24 + src/pkg/sync/atomic/asm_amd64.s | 29 + src/pkg/sync/atomic/asm_linux_arm.s | 27 + src/pkg/sync/atomic/atomic_test.go | 366 +++++- src/pkg/sync/atomic/doc.go | 25 + src/pkg/syscall/Makefile | 9 + src/pkg/syscall/asm_linux_386.s | 14 +- src/pkg/syscall/asm_openbsd_386.s | 137 ++ src/pkg/syscall/asm_openbsd_amd64.s | 130 ++ src/pkg/syscall/mkall.sh | 14 +- src/pkg/syscall/mkerrors.sh | 24 +- src/pkg/syscall/mksyscall.pl | 14 + src/pkg/syscall/mksysnum_openbsd.pl | 43 + src/pkg/syscall/netlink_linux.go | 6 +- src/pkg/syscall/route_bsd.go | 49 +- src/pkg/syscall/route_openbsd.go | 35 + src/pkg/syscall/syscall_openbsd.go | 261 ++++ src/pkg/syscall/syscall_openbsd_386.go | 42 + src/pkg/syscall/syscall_openbsd_amd64.go | 42 + src/pkg/syscall/syscall_windows.go | 36 +- src/pkg/syscall/types_linux.c | 2 +- src/pkg/syscall/types_openbsd.c | 182 +++ src/pkg/syscall/zerrors_openbsd_386.go | 1367 ++++++++++++++++++++ src/pkg/syscall/zerrors_openbsd_amd64.go | 1367 ++++++++++++++++++++ src/pkg/syscall/zsyscall_openbsd_386.go | 941 ++++++++++++++ src/pkg/syscall/zsyscall_openbsd_amd64.go | 941 ++++++++++++++ src/pkg/syscall/zsyscall_windows_386.go | 19 +- src/pkg/syscall/zsyscall_windows_amd64.go | 19 +- src/pkg/syscall/zsysnum_openbsd_386.go | 207 +++ src/pkg/syscall/zsysnum_openbsd_amd64.go | 204 +++ src/pkg/syscall/ztypes_linux_386.go | 2 +- src/pkg/syscall/ztypes_linux_amd64.go | 2 +- src/pkg/syscall/ztypes_linux_arm.go | 2 +- src/pkg/syscall/ztypes_openbsd_386.go | 383 ++++++ src/pkg/syscall/ztypes_openbsd_amd64.go | 390 ++++++ src/pkg/syscall/ztypes_windows.go | 25 +- src/pkg/template/doc.go | 2 +- src/pkg/template/exec.go | 63 +- src/pkg/template/exec_test.go | 35 +- src/pkg/template/parse/lex.go | 84 +- src/pkg/template/parse/lex_test.go | 14 + src/pkg/template/parse/node.go | 105 +- src/pkg/template/parse/parse.go | 8 +- src/pkg/template/parse/set.go | 2 +- src/pkg/time/Makefile | 10 +- src/pkg/time/internal_test.go | 12 + src/pkg/time/sleep_test.go | 3 +- src/pkg/time/sys_plan9.go | 5 + src/pkg/time/sys_posix.go | 18 - src/pkg/time/sys_unix.go | 23 + src/pkg/time/sys_windows.go | 22 + src/pkg/time/time_test.go | 8 - src/pkg/time/zoneinfo_plan9.go | 16 + src/pkg/time/zoneinfo_unix.go | 5 + src/pkg/time/zoneinfo_windows.go | 44 +- src/pkg/unsafe/unsafe.go | 6 +- src/pkg/url/url.go | 30 +- src/pkg/url/url_test.go | 62 +- src/pkg/websocket/Makefile | 2 + src/pkg/websocket/client.go | 309 +---- src/pkg/websocket/hixie.go | 694 ++++++++++ src/pkg/websocket/hixie_test.go | 201 +++ src/pkg/websocket/hybi.go | 515 ++++++++ src/pkg/websocket/hybi_test.go | 371 ++++++ src/pkg/websocket/server.go | 203 +-- src/pkg/websocket/websocket.go | 444 +++++-- src/pkg/websocket/websocket_test.go | 171 ++- src/pkg/xml/marshal.go | 93 +- src/pkg/xml/marshal_test.go | 123 ++ src/sudo.bash | 5 + src/version.bash | 2 +- 386 files changed, 22219 insertions(+), 4312 deletions(-) create mode 100644 src/cmd/gc/esc.c mode change 100755 => 100644 src/cmd/godoc/snippet.go create mode 100644 src/cmd/godoc/throttle.go create mode 100644 src/cmd/gofix/netudpgroup.go create mode 100644 src/cmd/gofix/netudpgroup_test.go create mode 100644 src/cmd/goinstall/tag_test.go mode change 100755 => 100644 src/pkg/big/int.go mode change 100755 => 100644 src/pkg/big/int_test.go mode change 100755 => 100644 src/pkg/big/nat.go mode change 100755 => 100644 src/pkg/big/nat_test.go mode change 100755 => 100644 src/pkg/crypto/rand/rand_windows.go create mode 100644 src/pkg/exp/norm/normalize_test.go create mode 100644 src/pkg/exp/norm/readwriter.go create mode 100644 src/pkg/exp/norm/readwriter_test.go create mode 100644 src/pkg/exp/regexp/exec_test.go create mode 100644 src/pkg/exp/regexp/re2.txt.gz create mode 100644 src/pkg/exp/template/html/js.go create mode 100644 src/pkg/exp/template/html/js_test.go create mode 100644 src/pkg/go/build/pkgtest/sqrt_386_test.go create mode 100644 src/pkg/go/build/pkgtest/sqrt_amd64_test.go create mode 100644 src/pkg/go/build/pkgtest/sqrt_arm_test.go create mode 100644 src/pkg/go/build/pkgtest/sqrt_test.go create mode 100644 src/pkg/go/build/pkgtest/xsqrt_test.go create mode 100644 src/pkg/http/filetransport.go create mode 100644 src/pkg/http/filetransport_test.go create mode 100644 src/pkg/image/png/testdata/invalid-crc32.png create mode 100644 src/pkg/image/png/testdata/invalid-noend.png create mode 100644 src/pkg/image/png/testdata/invalid-trunc.png create mode 100644 src/pkg/image/png/testdata/invalid-zlib.png create mode 100644 src/pkg/image/tiff/testdata/video-001-uncompressed.tiff create mode 100644 src/pkg/json/tags.go create mode 100644 src/pkg/json/tags_test.go mode change 100755 => 100644 src/pkg/math/atan2_decl.go delete mode 100644 src/pkg/mime/mime_test.go create mode 100644 src/pkg/mime/type_test.go create mode 100644 src/pkg/runtime/openbsd/386/defs.h create mode 100644 src/pkg/runtime/openbsd/386/rt0.s create mode 100644 src/pkg/runtime/openbsd/386/signal.c create mode 100644 src/pkg/runtime/openbsd/386/sys.s create mode 100644 src/pkg/runtime/syscall_windows_test.go create mode 100644 src/pkg/runtime/windows/386/callback.c create mode 100644 src/pkg/runtime/windows/amd64/callback.c create mode 100644 src/pkg/sort/export_test.go create mode 100644 src/pkg/syscall/asm_openbsd_386.s create mode 100644 src/pkg/syscall/asm_openbsd_amd64.s create mode 100755 src/pkg/syscall/mksysnum_openbsd.pl create mode 100644 src/pkg/syscall/route_openbsd.go create mode 100644 src/pkg/syscall/syscall_openbsd.go create mode 100644 src/pkg/syscall/syscall_openbsd_386.go create mode 100644 src/pkg/syscall/syscall_openbsd_amd64.go create mode 100644 src/pkg/syscall/types_openbsd.c create mode 100644 src/pkg/syscall/zerrors_openbsd_386.go create mode 100644 src/pkg/syscall/zerrors_openbsd_amd64.go create mode 100644 src/pkg/syscall/zsyscall_openbsd_386.go create mode 100644 src/pkg/syscall/zsyscall_openbsd_amd64.go create mode 100644 src/pkg/syscall/zsysnum_openbsd_386.go create mode 100644 src/pkg/syscall/zsysnum_openbsd_amd64.go create mode 100644 src/pkg/syscall/ztypes_openbsd_386.go create mode 100644 src/pkg/syscall/ztypes_openbsd_amd64.go create mode 100644 src/pkg/time/internal_test.go delete mode 100644 src/pkg/time/sys_posix.go create mode 100644 src/pkg/time/sys_unix.go create mode 100644 src/pkg/time/sys_windows.go create mode 100644 src/pkg/websocket/hixie.go create mode 100644 src/pkg/websocket/hixie_test.go create mode 100644 src/pkg/websocket/hybi.go create mode 100644 src/pkg/websocket/hybi_test.go (limited to 'src') diff --git a/src/Make.inc b/src/Make.inc index 8f549f624..c770b7632 100644 --- a/src/Make.inc +++ b/src/Make.inc @@ -33,6 +33,7 @@ GOOS_LIST=\ darwin\ freebsd\ linux\ + openbsd\ plan9\ windows\ @@ -117,7 +118,10 @@ HOST_AR?=ar HOST_EXTRA_CFLAGS?=-ggdb -O2 HOST_EXTRA_LDFLAGS?= -HOST_CFLAGS=-I"$(GOROOT)/include" $(HOST_EXTRA_CFLAGS) +# The -fno-common here is not necessary, but some compilers +# on OS X seem to set it by default. Setting it here keeps the build +# working in that non-standard context. +HOST_CFLAGS=-fno-common -I"$(GOROOT)/include" $(HOST_EXTRA_CFLAGS) HOST_LDFLAGS=$(HOST_EXTRA_LDFLAGS) PWD=$(shell pwd) diff --git a/src/Make.pkg b/src/Make.pkg index fc80cf6e6..ad7d10beb 100644 --- a/src/Make.pkg +++ b/src/Make.pkg @@ -83,10 +83,10 @@ $(TARGDIR)/$(TARG).a: _obj/$(TARG).a cp _obj/$(TARG).a "$@" _go_.$O: $(GOFILES) $(PREREQ) - $(GC) $(GCIMPORTS) -o $@ $(GOFILES) + $(GC) $(GCIMPORTS) -p $(TARG) -o $@ $(GOFILES) _gotest_.$O: $(GOFILES) $(GOTESTFILES) $(PREREQ) - $(GC) $(GCIMPORTS) -o $@ $(GOFILES) $(GOTESTFILES) + $(GC) $(GCIMPORTS) -p $(TARG) -o $@ $(GOFILES) $(GOTESTFILES) _obj/$(TARG).a: _go_.$O $(OFILES) @mkdir -p _obj/$(dir) diff --git a/src/cmd/5a/lex.c b/src/cmd/5a/lex.c index ad7ed05dd..4bef0219a 100644 --- a/src/cmd/5a/lex.c +++ b/src/cmd/5a/lex.c @@ -44,7 +44,11 @@ enum int systemtype(int sys) { +#ifdef _WIN32 + return sys&Windows; +#else return sys&Plan9; +#endif } void @@ -643,17 +647,13 @@ outhist(void) for(h = hist; h != H; h = h->link) { p = h->name; op = 0; - /* on windows skip drive specifier in pathname */ if(systemtype(Windows) && p && p[1] == ':'){ - p += 2; - c = *p; - } - if(p && p[0] != c && h->offset == 0 && pathname){ - /* on windows skip drive specifier in pathname */ + c = p[2]; + } else if(p && p[0] != c && h->offset == 0 && pathname){ if(systemtype(Windows) && pathname[1] == ':') { op = p; - p = pathname+2; - c = *p; + p = pathname; + c = p[2]; } else if(pathname[0] == c){ op = p; p = pathname; diff --git a/src/cmd/5c/gc.h b/src/cmd/5c/gc.h index ff6d51916..5349114f8 100644 --- a/src/cmd/5c/gc.h +++ b/src/cmd/5c/gc.h @@ -181,7 +181,7 @@ EXTERN int32 maxargsafe; EXTERN int mnstring; EXTERN Multab multab[20]; EXTERN int retok; -EXTERN int hintabsize; +extern int hintabsize; EXTERN Node* nodrat; EXTERN Node* nodret; EXTERN Node* nodsafe; diff --git a/src/cmd/5c/swt.c b/src/cmd/5c/swt.c index 7cbaadba9..32032532f 100644 --- a/src/cmd/5c/swt.c +++ b/src/cmd/5c/swt.c @@ -460,17 +460,13 @@ outhist(Biobuf *b) for(h = hist; h != H; h = h->link) { p = h->name; op = 0; - /* on windows skip drive specifier in pathname */ if(systemtype(Windows) && p && p[1] == ':'){ - p += 2; - c = *p; - } - if(p && p[0] != c && h->offset == 0 && pathname){ - /* on windows skip drive specifier in pathname */ + c = p[2]; + } else if(p && p[0] != c && h->offset == 0 && pathname){ if(systemtype(Windows) && pathname[1] == ':') { op = p; - p = pathname+2; - c = *p; + p = pathname; + c = p[2]; } else if(pathname[0] == c){ op = p; p = pathname; diff --git a/src/cmd/5g/cgen.c b/src/cmd/5g/cgen.c index 6e2fbe20f..0ea8695a0 100644 --- a/src/cmd/5g/cgen.c +++ b/src/cmd/5g/cgen.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "gg.h" /* @@ -850,6 +852,7 @@ bgen(Node *n, int true, Prog *to) Node n1, n2, n3, n4, tmp; Prog *p1, *p2; + USED(n4); // in unreachable code below if(debug['g']) { dump("\nbgen", n); } @@ -860,9 +863,6 @@ bgen(Node *n, int true, Prog *to) if(n->ninit != nil) genlist(n->ninit); - nl = n->left; - nr = n->right; - if(n->type == T) { convlit(&n, types[TBOOL]); if(n->type == T) @@ -875,7 +875,6 @@ bgen(Node *n, int true, Prog *to) patch(gins(AEND, N, N), to); goto ret; } - nl = N; nr = N; switch(n->op) { @@ -984,6 +983,7 @@ bgen(Node *n, int true, Prog *to) regfree(&n1); break; +#ifdef NOTDEF a = optoas(a, types[tptr]); regalloc(&n1, types[tptr], N); regalloc(&n3, types[tptr], N); @@ -1001,6 +1001,7 @@ bgen(Node *n, int true, Prog *to) regfree(&n3); regfree(&n1); break; +#endif } if(isinter(nl->type)) { @@ -1019,6 +1020,7 @@ bgen(Node *n, int true, Prog *to) regfree(&n1); break; +#ifdef NOTDEF a = optoas(a, types[tptr]); regalloc(&n1, types[tptr], N); regalloc(&n3, types[tptr], N); @@ -1036,6 +1038,7 @@ bgen(Node *n, int true, Prog *to) regfree(&n3); regfree(&n4); break; +#endif } if(iscomplex[nl->type->etype]) { @@ -1198,8 +1201,6 @@ sgen(Node *n, Node *res, int32 w) dump("r", n); dump("res", res); } - if(w == 0) - return; if(w < 0) fatal("sgen copy %d", w); if(n->ullman >= UINF && res->ullman >= UINF) @@ -1207,12 +1208,20 @@ sgen(Node *n, Node *res, int32 w) if(n->type == T) fatal("sgen: missing type"); + if(w == 0) { + // evaluate side effects only. + regalloc(&dst, types[tptr], N); + agen(res, &dst); + agen(n, &dst); + regfree(&dst); + return; + } + // determine alignment. // want to avoid unaligned access, so have to use // smaller operations for less aligned types. // for example moving [4]byte must use 4 MOVB not 1 MOVW. align = n->type->align; - op = 0; switch(align) { default: fatal("sgen: invalid alignment %d for %T", align, n->type); @@ -1313,7 +1322,6 @@ sgen(Node *n, Node *res, int32 w) p->from.type = D_OREG; p->from.offset = dir; p->scond |= C_PBIT; - ploop = p; p = gins(op, &tmp, &dst); p->to.type = D_OREG; diff --git a/src/cmd/5g/cgen64.c b/src/cmd/5g/cgen64.c index b56df765b..1235d1ace 100644 --- a/src/cmd/5g/cgen64.c +++ b/src/cmd/5g/cgen64.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "gg.h" /* @@ -240,7 +242,7 @@ cgen64(Node *n, Node *res) // shift is >= 1<<32 split64(r, &cl, &ch); gmove(&ch, &s); - p1 = gins(ATST, &s, N); + gins(ATST, &s, N); p6 = gbranch(ABNE, T); gmove(&cl, &s); splitclean(); @@ -248,7 +250,7 @@ cgen64(Node *n, Node *res) gmove(r, &s); p6 = P; } - p1 = gins(ATST, &s, N); + gins(ATST, &s, N); // shift == 0 p1 = gins(AMOVW, &bl, &al); @@ -411,7 +413,7 @@ olsh_break: gmove(r, &s); p6 = P; } - p1 = gins(ATST, &s, N); + gins(ATST, &s, N); // shift == 0 p1 = gins(AMOVW, &bl, &al); @@ -453,9 +455,9 @@ olsh_break: p1 = gins(AMOVW, &bh, &al); p1->scond = C_SCOND_EQ; if(bh.type->etype == TINT32) - p1 = gshift(AMOVW, &bh, SHIFT_AR, 31, &ah); + gshift(AMOVW, &bh, SHIFT_AR, 31, &ah); else - p1 = gins(AEOR, &ah, &ah); + gins(AEOR, &ah, &ah); p4 = gbranch(ABEQ, T); // check if shift is < 64 diff --git a/src/cmd/5g/galign.c b/src/cmd/5g/galign.c index 12766102f..070804217 100644 --- a/src/cmd/5g/galign.c +++ b/src/cmd/5g/galign.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "gg.h" int thechar = '5'; diff --git a/src/cmd/5g/gg.h b/src/cmd/5g/gg.h index ce4558e21..c826d2652 100644 --- a/src/cmd/5g/gg.h +++ b/src/cmd/5g/gg.h @@ -2,16 +2,13 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include -#include +#ifndef EXTERN +#define EXTERN extern +#endif #include "../gc/go.h" #include "../5l/5.out.h" -#ifndef EXTERN -#define EXTERN extern -#endif - typedef struct Addr Addr; struct Addr @@ -51,22 +48,19 @@ struct Prog #define REGALLOC_F0 (REGALLOC_RMAX+1) #define REGALLOC_FMAX (REGALLOC_F0 + FREGEXT) -EXTERN Biobuf* bout; EXTERN int32 dynloc; EXTERN uchar reg[REGALLOC_FMAX+1]; EXTERN int32 pcloc; // instruction counter EXTERN Strlit emptystring; extern char* anames[]; -EXTERN Hist* hist; EXTERN Prog zprog; -EXTERN Node* curfn; EXTERN Node* newproc; EXTERN Node* deferproc; EXTERN Node* deferreturn; EXTERN Node* panicindex; EXTERN Node* panicslice; EXTERN Node* throwreturn; -EXTERN long unmappedzero; +extern long unmappedzero; EXTERN int maxstksize; /* @@ -169,3 +163,6 @@ int Yconv(Fmt*); void listinit(void); void zaddr(Biobuf*, Addr*, int); + +#pragma varargck type "D" Addr* +#pragma varargck type "M" Addr* diff --git a/src/cmd/5g/ggen.c b/src/cmd/5g/ggen.c index 3f5f47e7b..3f38318e7 100644 --- a/src/cmd/5g/ggen.c +++ b/src/cmd/5g/ggen.c @@ -4,6 +4,8 @@ #undef EXTERN #define EXTERN +#include +#include #include "gg.h" #include "opt.h" @@ -544,7 +546,7 @@ cgen_shift(int op, Node *nl, Node *nr, Node *res) } // test for shift being 0 - p1 = gins(ATST, &n1, N); + gins(ATST, &n1, N); p3 = gbranch(ABEQ, T); // test and fix up large shifts diff --git a/src/cmd/5g/gobj.c b/src/cmd/5g/gobj.c index 27c8be67d..9f728dee7 100644 --- a/src/cmd/5g/gobj.c +++ b/src/cmd/5g/gobj.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "gg.h" void @@ -266,54 +268,6 @@ dumpfuncs(void) } } -/* deferred DATA output */ -static Prog *strdat; -static Prog *estrdat; -static int gflag; -static Prog *savepc; - -void -data(void) -{ - gflag = debug['g']; - debug['g'] = 0; - - if(estrdat == nil) { - strdat = mal(sizeof(*pc)); - clearp(strdat); - estrdat = strdat; - } - if(savepc) - fatal("data phase error"); - savepc = pc; - pc = estrdat; -} - -void -text(void) -{ - if(!savepc) - fatal("text phase error"); - debug['g'] = gflag; - estrdat = pc; - pc = savepc; - savepc = nil; -} - -void -dumpdata(void) -{ - Prog *p; - - if(estrdat == nil) - return; - *pc = *strdat; - if(gflag) - for(p=pc; p!=estrdat; p=p->link) - print("%P\n", p); - pc = estrdat; -} - int dsname(Sym *sym, int off, char *t, int n) { @@ -379,6 +333,17 @@ gdata(Node *nam, Node *nr, int wid) Prog *p; vlong v; + if(nr->op == OLITERAL) { + switch(nr->val.ctype) { + case CTCPLX: + gdatacomplex(nam, nr->val.u.cval); + return; + case CTSTR: + gdatastring(nam, nr->val.u.sval); + return; + } + } + if(wid == 8 && is64(nr->type)) { v = mpgetfix(nr->val.u.xval); p = gins(ADATA, nam, nodintconst(v)); diff --git a/src/cmd/5g/gsubr.c b/src/cmd/5g/gsubr.c index ddaf52a88..f8920df87 100644 --- a/src/cmd/5g/gsubr.c +++ b/src/cmd/5g/gsubr.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "gg.h" // TODO(kaib): Can make this bigger if we move @@ -50,6 +52,10 @@ clearp(Prog *p) pcloc++; } +static int ddumped; +static Prog *dfirst; +static Prog *dpc; + /* * generate and return proc with p->as = as, * linked into program. pc is next instruction. @@ -59,10 +65,22 @@ prog(int as) { Prog *p; - p = pc; - pc = mal(sizeof(*pc)); - - clearp(pc); + if(as == ADATA || as == AGLOBL) { + if(ddumped) + fatal("already dumped data"); + if(dpc == nil) { + dpc = mal(sizeof(*dpc)); + dfirst = dpc; + } + p = dpc; + dpc = mal(sizeof(*dpc)); + p->link = dpc; + } else { + p = pc; + pc = mal(sizeof(*pc)); + clearp(pc); + p->link = pc; + } if(lineno == 0) { if(debug['K']) @@ -71,10 +89,21 @@ prog(int as) p->as = as; p->lineno = lineno; - p->link = pc; return p; } +void +dumpdata(void) +{ + ddumped = 1; + if(dfirst == nil) + return; + newplist(); + *pc = *dfirst; + pc = dpc; + clearp(pc); +} + /* * generate a branch. * t is ignored. @@ -84,6 +113,8 @@ gbranch(int as, Type *t) { Prog *p; + USED(t); + p = prog(as); p->to.type = D_BRANCH; p->to.branch = P; @@ -1242,7 +1273,6 @@ naddr(Node *n, Addr *a, int canemitcode) a->etype = simtype[n->type->etype]; a->width = n->type->width; } - a->pun = n->pun; a->offset = n->xoffset; a->sym = n->sym; if(a->sym == S) diff --git a/src/cmd/5g/list.c b/src/cmd/5g/list.c index 0c6dbbf71..fef9c8543 100644 --- a/src/cmd/5g/list.c +++ b/src/cmd/5g/list.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "gg.h" // TODO(kaib): make 5g/list.c congruent with 5l/list.c diff --git a/src/cmd/5g/peep.c b/src/cmd/5g/peep.c index 6cc93db12..e87f5d697 100644 --- a/src/cmd/5g/peep.c +++ b/src/cmd/5g/peep.c @@ -29,6 +29,8 @@ // THE SOFTWARE. +#include +#include #include "gg.h" #include "opt.h" @@ -45,6 +47,9 @@ peep(void) Reg *r, *r1, *r2; Prog *p, *p1; int t; + + p1 = nil; + USED(p1); // ... in unreachable code... /* * complete R structure */ @@ -115,12 +120,14 @@ loop1: } break; +#ifdef NOTDEF if(p->scond == C_SCOND_NONE) if(regtyp(&p->to)) if(isdconst(&p->from)) { constprop(&p->from, &p->to, r->s1); } break; +#endif } } if(t) @@ -128,6 +135,7 @@ loop1: return; +#ifdef NOTDEF for(r=firstr; r!=R; r=r->link) { p = r->prog; switch(p->as) { @@ -255,6 +263,7 @@ return; // } predicate(); +#endif } Reg* @@ -1159,7 +1168,6 @@ copyu(Prog *p, Adr *v, Adr *s) return 3; return 0; } - return 0; } /* diff --git a/src/cmd/5g/reg.c b/src/cmd/5g/reg.c index 2d2a6d01a..9dd3f07f1 100644 --- a/src/cmd/5g/reg.c +++ b/src/cmd/5g/reg.c @@ -29,6 +29,8 @@ // THE SOFTWARE. +#include +#include #include "gg.h" #include "opt.h" @@ -112,19 +114,19 @@ setaddrs(Bits bit) { int i, n; Var *v; - Sym *s; + Node *node; while(bany(&bit)) { // convert each bit to a variable i = bnum(bit); - s = var[i].sym; + node = var[i].node; n = var[i].name; bit.b[i/32] &= ~(1L<<(i%32)); // disable all pieces of that variable for(i=0; isym == s && v->name == n) + if(v->node == node && v->name == n) v->addr = 2; } } @@ -202,7 +204,7 @@ regopt(Prog *firstp) nvar = NREGVAR; memset(var, 0, NREGVAR*sizeof var[0]); for(i=0; ito; - a->sym = v->sym; a->name = v->name; a->node = v->node; + a->sym = v->node->sym; a->offset = v->offset; a->etype = v->etype; a->type = D_OREG; @@ -838,11 +840,10 @@ mkvar(Reg *r, Adr *a) int i, t, n, et, z, w, flag; int32 o; Bits bit; - Sym *s; + Node *node; // mark registers used t = a->type; - n = D_NONE; flag = 0; switch(t) { @@ -909,10 +910,11 @@ mkvar(Reg *r, Adr *a) break; } - s = a->sym; - if(s == S) + node = a->node; + if(node == N || node->op != ONAME || node->orig != N) goto none; - if(s->name[0] == '.') + node = node->orig; + if(node->sym->name[0] == '.') goto none; et = a->etype; o = a->offset; @@ -920,7 +922,7 @@ mkvar(Reg *r, Adr *a) for(i=0; isym == s && v->name == n) { + if(v->node == node && v->name == n) { if(v->offset == o) if(v->etype == et) if(v->width == w) @@ -944,7 +946,7 @@ mkvar(Reg *r, Adr *a) } if(nvar >= NVAR) { - if(debug['w'] > 1 && s) + if(debug['w'] > 1 && node) fatal("variable not optimized: %D", a); goto none; } @@ -953,17 +955,16 @@ mkvar(Reg *r, Adr *a) nvar++; //print("var %d %E %D %S\n", i, et, a, s); v = var+i; - v->sym = s; v->offset = o; v->name = n; // v->gotype = a->gotype; v->etype = et; v->width = w; v->addr = flag; // funny punning - v->node = a->node; + v->node = node; if(debug['R']) - print("bit=%2d et=%2d w=%d+%d %S %D flag=%d\n", i, et, o, w, s, a, v->addr); + print("bit=%2d et=%2d w=%d+%d %#N %D flag=%d\n", i, et, o, w, node, a, v->addr); bit = blsh(i); if(n == D_EXTERN || n == D_STATIC) diff --git a/src/cmd/5l/softfloat.c b/src/cmd/5l/softfloat.c index 4f799d17e..401107178 100644 --- a/src/cmd/5l/softfloat.c +++ b/src/cmd/5l/softfloat.c @@ -2,7 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#define EXTERN #include "l.h" #include "../ld/lib.h" diff --git a/src/cmd/6a/lex.c b/src/cmd/6a/lex.c index 42f4b1d11..1a8e5ad61 100644 --- a/src/cmd/6a/lex.c +++ b/src/cmd/6a/lex.c @@ -44,7 +44,11 @@ enum int systemtype(int sys) { +#ifdef _WIN32 + return sys&Windows; +#else return sys&Plan9; +#endif } int @@ -456,8 +460,8 @@ struct "JGT", LTYPER, AJGT, "JG", LTYPER, AJGT, /* alternate */ "JNLE", LTYPER, AJGT, /* alternate */ - - "JCXZ", LTYPER, AJCXZ, + "JCXZL", LTYPER, AJCXZL, + "JCXZQ", LTYPER, AJCXZQ, "JMP", LTYPEC, AJMP, "LAHF", LTYPE0, ALAHF, "LARL", LTYPE3, ALARL, @@ -1251,17 +1255,13 @@ outhist(void) for(h = hist; h != H; h = h->link) { p = h->name; op = 0; - /* on windows skip drive specifier in pathname */ if(systemtype(Windows) && p && p[1] == ':'){ - p += 2; - c = *p; - } - if(p && p[0] != c && h->offset == 0 && pathname){ - /* on windows skip drive specifier in pathname */ + c = p[2]; + } else if(p && p[0] != c && h->offset == 0 && pathname){ if(systemtype(Windows) && pathname[1] == ':') { op = p; - p = pathname+2; - c = *p; + p = pathname; + c = p[2]; } else if(pathname[0] == c){ op = p; p = pathname; diff --git a/src/cmd/6c/swt.c b/src/cmd/6c/swt.c index d7a917043..3de86306d 100644 --- a/src/cmd/6c/swt.c +++ b/src/cmd/6c/swt.c @@ -331,17 +331,13 @@ outhist(Biobuf *b) for(h = hist; h != H; h = h->link) { p = h->name; op = 0; - /* on windows skip drive specifier in pathname */ if(systemtype(Windows) && p && p[1] == ':'){ - p += 2; - c = *p; - } - if(p && p[0] != c && h->offset == 0 && pathname){ - /* on windows skip drive specifier in pathname */ + c = p[2]; + } else if(p && p[0] != c && h->offset == 0 && pathname){ if(systemtype(Windows) && pathname[1] == ':') { op = p; - p = pathname+2; - c = *p; + p = pathname; + c = p[2]; } else if(pathname[0] == c){ op = p; p = pathname; diff --git a/src/cmd/6g/cgen.c b/src/cmd/6g/cgen.c index 24f88a416..43bec0059 100644 --- a/src/cmd/6g/cgen.c +++ b/src/cmd/6g/cgen.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "gg.h" /* @@ -727,9 +729,6 @@ bgen(Node *n, int true, Prog *to) if(n->ninit != nil) genlist(n->ninit); - nl = n->left; - nr = n->right; - if(n->type == T) { convlit(&n, types[TBOOL]); if(n->type == T) @@ -742,7 +741,6 @@ bgen(Node *n, int true, Prog *to) patch(gins(AEND, N, N), to); goto ret; } - nl = N; nr = N; switch(n->op) { @@ -1031,11 +1029,9 @@ sgen(Node *n, Node *ns, int32 w) dump("r", n); dump("res", ns); } - if(w == 0) - return; - if(n->ullman >= UINF && ns->ullman >= UINF) { + + if(n->ullman >= UINF && ns->ullman >= UINF) fatal("sgen UINF"); - } if(w < 0) fatal("sgen copy %d", w); @@ -1043,6 +1039,15 @@ sgen(Node *n, Node *ns, int32 w) if(w == 16) if(componentgen(n, ns)) return; + + if(w == 0) { + // evaluate side effects only + regalloc(&nodr, types[tptr], N); + agen(ns, &nodr); + agen(n, &nodr); + regfree(&nodr); + return; + } // offset on the stack osrc = stkof(n); diff --git a/src/cmd/6g/galign.c b/src/cmd/6g/galign.c index e366362b3..b03ac1ed6 100644 --- a/src/cmd/6g/galign.c +++ b/src/cmd/6g/galign.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "gg.h" int thechar = '6'; diff --git a/src/cmd/6g/gg.h b/src/cmd/6g/gg.h index 2493771a0..8a80ee9fb 100644 --- a/src/cmd/6g/gg.h +++ b/src/cmd/6g/gg.h @@ -2,16 +2,13 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include -#include +#ifndef EXTERN +#define EXTERN extern +#endif #include "../gc/go.h" #include "../6l/6.out.h" -#ifndef EXTERN -#define EXTERN extern -#endif - typedef struct Addr Addr; struct Addr @@ -44,22 +41,19 @@ struct Prog void* reg; // pointer to containing Reg struct }; -EXTERN Biobuf* bout; EXTERN int32 dynloc; EXTERN uchar reg[D_NONE]; EXTERN int32 pcloc; // instruction counter EXTERN Strlit emptystring; extern char* anames[]; -EXTERN Hist* hist; EXTERN Prog zprog; -EXTERN Node* curfn; EXTERN Node* newproc; EXTERN Node* deferproc; EXTERN Node* deferreturn; EXTERN Node* panicindex; EXTERN Node* panicslice; EXTERN Node* throwreturn; -EXTERN vlong unmappedzero; +extern vlong unmappedzero; /* * gen.c @@ -159,3 +153,5 @@ void listinit(void); void zaddr(Biobuf*, Addr*, int, int); +#pragma varargck type "D" Addr* +#pragma varargck type "lD" Addr* diff --git a/src/cmd/6g/ggen.c b/src/cmd/6g/ggen.c index a5f278384..2f68885bd 100644 --- a/src/cmd/6g/ggen.c +++ b/src/cmd/6g/ggen.c @@ -4,6 +4,8 @@ #undef EXTERN #define EXTERN +#include +#include #include "gg.h" #include "opt.h" @@ -446,8 +448,8 @@ dodiv(int op, Node *nl, Node *nr, Node *res) { int a, check; Node n3, n4, n5; - Type *t; - Node ax, dx, oldax, olddx; + Type *t, *t0; + Node ax, dx, ax1, n31, oldax, olddx; Prog *p1, *p2, *p3; // Have to be careful about handling @@ -459,6 +461,7 @@ dodiv(int op, Node *nl, Node *nr, Node *res) // For int32 and int64, use explicit test. // Could use int64 hw for int32. t = nl->type; + t0 = t; check = 0; if(issigned[t->etype]) { check = 1; @@ -476,18 +479,28 @@ dodiv(int op, Node *nl, Node *nr, Node *res) } a = optoas(op, t); - regalloc(&n3, t, N); + regalloc(&n3, t0, N); if(nl->ullman >= nr->ullman) { - savex(D_AX, &ax, &oldax, res, t); + savex(D_AX, &ax, &oldax, res, t0); cgen(nl, &ax); - regalloc(&ax, t, &ax); // mark ax live during cgen + regalloc(&ax, t0, &ax); // mark ax live during cgen cgen(nr, &n3); regfree(&ax); } else { cgen(nr, &n3); - savex(D_AX, &ax, &oldax, res, t); + savex(D_AX, &ax, &oldax, res, t0); cgen(nl, &ax); } + if(t != t0) { + // Convert + ax1 = ax; + n31 = n3; + ax.type = t; + n3.type = t; + gmove(&ax1, &ax); + gmove(&n31, &n3); + } + p3 = P; if(check) { nodconst(&n4, t, -1); diff --git a/src/cmd/6g/gobj.c b/src/cmd/6g/gobj.c index ba8a4870e..4dcce39c8 100644 --- a/src/cmd/6g/gobj.c +++ b/src/cmd/6g/gobj.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "gg.h" void @@ -278,54 +280,6 @@ dumpfuncs(void) } } -/* deferred DATA output */ -static Prog *strdat; -static Prog *estrdat; -static int gflag; -static Prog *savepc; - -void -data(void) -{ - gflag = debug['g']; - debug['g'] = 0; - - if(estrdat == nil) { - strdat = mal(sizeof(*pc)); - clearp(strdat); - estrdat = strdat; - } - if(savepc) - fatal("data phase error"); - savepc = pc; - pc = estrdat; -} - -void -text(void) -{ - if(!savepc) - fatal("text phase error"); - debug['g'] = gflag; - estrdat = pc; - pc = savepc; - savepc = nil; -} - -void -dumpdata(void) -{ - Prog *p; - - if(estrdat == nil) - return; - *pc = *strdat; - if(gflag) - for(p=pc; p!=estrdat; p=p->link) - print("%P\n", p); - pc = estrdat; -} - int dsname(Sym *s, int off, char *t, int n) { @@ -381,6 +335,16 @@ gdata(Node *nam, Node *nr, int wid) { Prog *p; + if(nr->op == OLITERAL) { + switch(nr->val.ctype) { + case CTCPLX: + gdatacomplex(nam, nr->val.u.cval); + return; + case CTSTR: + gdatastring(nam, nr->val.u.sval); + return; + } + } p = gins(ADATA, nam, nr); p->from.scale = wid; } @@ -537,6 +501,8 @@ genembedtramp(Type *rcvr, Type *method, Sym *newnam, int iface) int c, d, o, mov, add, loaded; Prog *p; Type *f; + + USED(iface); if(debug['r']) print("genembedtramp %T %T %S\n", rcvr, method, newnam); @@ -619,7 +585,6 @@ out: // but 6l has a bug, and it can't handle // JMP instructions too close to the top of // a new function. - p = pc; gins(ANOP, N, N); } diff --git a/src/cmd/6g/gsubr.c b/src/cmd/6g/gsubr.c index d0d6d0c96..92b15ef00 100644 --- a/src/cmd/6g/gsubr.c +++ b/src/cmd/6g/gsubr.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "gg.h" // TODO(rsc): Can make this bigger if we move @@ -46,6 +48,10 @@ clearp(Prog *p) pcloc++; } +static int ddumped; +static Prog *dfirst; +static Prog *dpc; + /* * generate and return proc with p->as = as, * linked into program. pc is next instruction. @@ -55,10 +61,22 @@ prog(int as) { Prog *p; - p = pc; - pc = mal(sizeof(*pc)); - - clearp(pc); + if(as == ADATA || as == AGLOBL) { + if(ddumped) + fatal("already dumped data"); + if(dpc == nil) { + dpc = mal(sizeof(*dpc)); + dfirst = dpc; + } + p = dpc; + dpc = mal(sizeof(*dpc)); + p->link = dpc; + } else { + p = pc; + pc = mal(sizeof(*pc)); + clearp(pc); + p->link = pc; + } if(lineno == 0) { if(debug['K']) @@ -67,10 +85,21 @@ prog(int as) p->as = as; p->lineno = lineno; - p->link = pc; return p; } +void +dumpdata(void) +{ + ddumped = 1; + if(dfirst == nil) + return; + newplist(); + *pc = *dfirst; + pc = dpc; + clearp(pc); +} + /* * generate a branch. * t is ignored. @@ -79,6 +108,8 @@ Prog* gbranch(int as, Type *t) { Prog *p; + + USED(t); p = prog(as); p->to.type = D_BRANCH; @@ -1098,7 +1129,6 @@ naddr(Node *n, Addr *a, int canemitcode) a->width = n->type->width; a->gotype = ngotype(n); } - a->pun = n->pun; a->offset = n->xoffset; a->sym = n->sym; if(a->sym == S) diff --git a/src/cmd/6g/list.c b/src/cmd/6g/list.c index c8077c97a..ad63f7d29 100644 --- a/src/cmd/6g/list.c +++ b/src/cmd/6g/list.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "gg.h" static int sconsize; @@ -131,7 +133,7 @@ Dconv(Fmt *fp) if(fp->flags & FmtLong) { d1 = a->offset & 0xffffffffLL; d2 = (a->offset>>32) & 0xffffffffLL; - snprint(str, sizeof(str), "$%ud-%ud", (ulong)d1, (ulong)d2); + snprint(str, sizeof(str), "$%lud-%lud", (ulong)d1, (ulong)d2); break; } snprint(str, sizeof(str), "$%lld", a->offset); diff --git a/src/cmd/6g/peep.c b/src/cmd/6g/peep.c index 4432203f2..63ef3f78f 100644 --- a/src/cmd/6g/peep.c +++ b/src/cmd/6g/peep.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "gg.h" #include "opt.h" @@ -991,7 +993,6 @@ loop: if(p->from.dval == p0->from.dval) if(p->from.index == p0->from.index) { excise(r); - t++; goto loop; } break; diff --git a/src/cmd/6g/reg.c b/src/cmd/6g/reg.c index 4d4263047..f380ced8c 100644 --- a/src/cmd/6g/reg.c +++ b/src/cmd/6g/reg.c @@ -28,9 +28,9 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "gg.h" -#undef EXTERN -#define EXTERN #include "opt.h" #define NREGVAR 32 /* 16 general + 16 floating */ @@ -98,19 +98,19 @@ setaddrs(Bits bit) { int i, n; Var *v; - Sym *s; + Node *node; while(bany(&bit)) { // convert each bit to a variable i = bnum(bit); - s = var[i].sym; + node = var[i].node; n = var[i].name; bit.b[i/32] &= ~(1L<<(i%32)); // disable all pieces of that variable for(i=0; isym == s && v->name == n) + if(v->node == node && v->name == n) v->addr = 2; } } @@ -179,7 +179,6 @@ regopt(Prog *firstp) r1 = R; firstr = R; lastr = R; - nvar = 0; /* * control flow is more complicated in generated go code @@ -189,7 +188,7 @@ regopt(Prog *firstp) nvar = NREGVAR; memset(var, 0, NREGVAR*sizeof var[0]); for(i=0; ito; - a->sym = v->sym; a->offset = v->offset; a->etype = v->etype; a->type = v->name; a->gotype = v->gotype; a->node = v->node; + a->sym = v->node->sym; // need to clean this up with wptr and // some of the defaults @@ -933,7 +932,7 @@ mkvar(Reg *r, Adr *a) uint32 regu; int32 o; Bits bit; - Sym *s; + Node *node; /* * mark registers used @@ -969,10 +968,11 @@ mkvar(Reg *r, Adr *a) n = t; break; } - s = a->sym; - if(s == S) + node = a->node; + if(node == N || node->op != ONAME || node->orig != N) goto none; - if(s->name[0] == '.') + node = node->orig; + if(node->sym->name[0] == '.') goto none; et = a->etype; o = a->offset; @@ -981,7 +981,7 @@ mkvar(Reg *r, Adr *a) flag = 0; for(i=0; isym == s && v->name == n) { + if(v->node == node && v->name == n) { if(v->offset == o) if(v->etype == et) if(v->width == w) @@ -995,11 +995,6 @@ mkvar(Reg *r, Adr *a) } } } - if(a->pun) { -// print("disable pun %s\n", s->name); - flag = 1; - - } switch(et) { case 0: case TFUNC: @@ -1007,25 +1002,24 @@ mkvar(Reg *r, Adr *a) } if(nvar >= NVAR) { - if(debug['w'] > 1 && s) - fatal("variable not optimized: %D", a); + if(debug['w'] > 1 && node != N) + fatal("variable not optimized: %#N", node); goto none; } i = nvar; nvar++; v = var+i; - v->sym = s; v->offset = o; v->name = n; v->gotype = a->gotype; v->etype = et; v->width = w; v->addr = flag; // funny punning - v->node = a->node; + v->node = node; if(debug['R']) - print("bit=%2d et=%2d w=%d %S %D\n", i, et, w, s, a); + print("bit=%2d et=%2d w=%d %#N %D\n", i, et, w, node, a); ostats.nvar++; bit = blsh(i); diff --git a/src/cmd/6l/6.out.h b/src/cmd/6l/6.out.h index 262da02ab..9a69c8ad5 100644 --- a/src/cmd/6l/6.out.h +++ b/src/cmd/6l/6.out.h @@ -121,7 +121,7 @@ enum as AIRETW, AJCC, AJCS, - AJCXZ, + AJCXZL, AJEQ, AJGE, AJGT, @@ -487,6 +487,7 @@ enum as AIDIVQ, AIMULQ, AIRETQ, + AJCXZQ, ALEAQ, ALEAVEQ, ALODSQ, diff --git a/src/cmd/6l/optab.c b/src/cmd/6l/optab.c index 36806ec4b..0a4c0eb07 100644 --- a/src/cmd/6l/optab.c +++ b/src/cmd/6l/optab.c @@ -788,7 +788,8 @@ Optab optab[] = { AIRETW, ynone, Pe, 0xcf }, { AJCC, yjcond, Px, 0x73,0x83,(00) }, { AJCS, yjcond, Px, 0x72,0x82 }, - { AJCXZ, yloop, Px, 0xe3 }, + { AJCXZL, yloop, Px, 0xe3 }, + { AJCXZQ, yloop, Px, 0xe3 }, { AJEQ, yjcond, Px, 0x74,0x84 }, { AJGE, yjcond, Px, 0x7d,0x8d }, { AJGT, yjcond, Px, 0x7f,0x8f }, diff --git a/src/cmd/6l/span.c b/src/cmd/6l/span.c index 5d13ad44b..9b869a493 100644 --- a/src/cmd/6l/span.c +++ b/src/cmd/6l/span.c @@ -88,7 +88,10 @@ span1(Sym *s) loop++; q->back ^= 2; } - s->p[q->pc+1] = v; + if(q->as == AJCXZL) + s->p[q->pc+2] = v; + else + s->p[q->pc+1] = v; } else { bp = s->p + q->pc + q->mark - 4; *bp++ = v; @@ -1439,10 +1442,11 @@ found: case Zbr: case Zjmp: + case Zloop: // TODO: jump across functions needs reloc q = p->pcond; if(q == nil) { - diag("jmp/branch without target"); + diag("jmp/branch/loop without target"); errorexit(); } if(q->as == ATEXT) { @@ -1466,8 +1470,12 @@ found: if(p->back & 1) { v = q->pc - (p->pc + 2); if(v >= -128) { + if(p->as == AJCXZL) + *andptr++ = 0x67; *andptr++ = op; *andptr++ = v; + } else if(t[2] == Zloop) { + diag("loop too far: %P", p); } else { v -= 5-2; if(t[2] == Zbr) { @@ -1487,8 +1495,12 @@ found: p->forwd = q->comefrom; q->comefrom = p; if(p->back & 2) { // short + if(p->as == AJCXZL) + *andptr++ = 0x67; *andptr++ = op; *andptr++ = 0; + } else if(t[2] == Zloop) { + diag("loop too far: %P", p); } else { if(t[2] == Zbr) *andptr++ = 0x0f; @@ -1520,19 +1532,6 @@ found: */ break; - case Zloop: - q = p->pcond; - if(q == nil) { - diag("loop without target"); - errorexit(); - } - v = q->pc - p->pc - 2; - if(v < -128 && v > 127) - diag("loop too far: %P", p); - *andptr++ = op; - *andptr++ = v; - break; - case Zbyte: v = vaddr(&p->from, &rel); if(rel.siz != 0) { diff --git a/src/cmd/8a/a.y b/src/cmd/8a/a.y index a8ac773da..96976089d 100644 --- a/src/cmd/8a/a.y +++ b/src/cmd/8a/a.y @@ -210,6 +210,13 @@ spec3: /* JMP/CALL */ $$.from = nullgen; $$.to = $1; } +| '*' nam + { + $$.from = nullgen; + $$.to = $2; + $$.to.index = $2.type; + $$.to.type = D_INDIR+D_ADDR; + } spec4: /* NOP */ nonnon diff --git a/src/cmd/8a/lex.c b/src/cmd/8a/lex.c index e56460e4b..ca2e2c138 100644 --- a/src/cmd/8a/lex.c +++ b/src/cmd/8a/lex.c @@ -44,7 +44,11 @@ enum int systemtype(int sys) { +#ifdef _WIN32 + return sys&Windows; +#else return sys&Plan9; +#endif } int @@ -371,7 +375,8 @@ struct "JG", LTYPER, AJGT, /* alternate */ "JNLE", LTYPER, AJGT, /* alternate */ - "JCXZ", LTYPER, AJCXZ, + "JCXZL", LTYPER, AJCXZL, + "JCXZW", LTYPER, AJCXZW, "JMP", LTYPEC, AJMP, "LAHF", LTYPE0, ALAHF, "LARL", LTYPE3, ALARL, @@ -911,17 +916,13 @@ outhist(void) for(h = hist; h != H; h = h->link) { p = h->name; op = 0; - /* on windows skip drive specifier in pathname */ if(systemtype(Windows) && p && p[1] == ':'){ - p += 2; - c = *p; - } - if(p && p[0] != c && h->offset == 0 && pathname){ - /* on windows skip drive specifier in pathname */ + c = p[2]; + } else if(p && p[0] != c && h->offset == 0 && pathname){ if(systemtype(Windows) && pathname[1] == ':') { op = p; - p = pathname+2; - c = *p; + p = pathname; + c = p[2]; } else if(pathname[0] == c){ op = p; p = pathname; diff --git a/src/cmd/8c/swt.c b/src/cmd/8c/swt.c index 769ef2c66..006bfdfe2 100644 --- a/src/cmd/8c/swt.c +++ b/src/cmd/8c/swt.c @@ -330,17 +330,13 @@ outhist(Biobuf *b) for(h = hist; h != H; h = h->link) { p = h->name; op = 0; - /* on windows skip drive specifier in pathname */ if(systemtype(Windows) && p && p[1] == ':'){ - p += 2; - c = *p; - } - if(p && p[0] != c && h->offset == 0 && pathname){ - /* on windows skip drive specifier in pathname */ + c = p[2]; + } else if(p && p[0] != c && h->offset == 0 && pathname){ if(systemtype(Windows) && pathname[1] == ':') { op = p; - p = pathname+2; - c = *p; + p = pathname; + c = p[2]; } else if(pathname[0] == c){ op = p; p = pathname; diff --git a/src/cmd/8g/cgen.c b/src/cmd/8g/cgen.c index b316e6e34..21b7815fd 100644 --- a/src/cmd/8g/cgen.c +++ b/src/cmd/8g/cgen.c @@ -5,6 +5,8 @@ // TODO(rsc): // assume CLD? +#include +#include #include "gg.h" void @@ -680,7 +682,6 @@ agen(Node *n, Node *res) break; case ODOT: - t = nl->type; agen(nl, res); if(n->xoffset != 0) { nodconst(&n1, types[tptr], n->xoffset); @@ -798,9 +799,6 @@ bgen(Node *n, int true, Prog *to) if(n->ninit != nil) genlist(n->ninit); - nl = n->left; - nr = n->right; - if(n->type == T) { convlit(&n, types[TBOOL]); if(n->type == T) @@ -813,7 +811,6 @@ bgen(Node *n, int true, Prog *to) patch(gins(AEND, N, N), to); return; } - nl = N; nr = N; switch(n->op) { @@ -1139,15 +1136,20 @@ sgen(Node *n, Node *res, int32 w) dump("r", n); dump("res", res); } - if(w == 0) - return; - if(n->ullman >= UINF && res->ullman >= UINF) { + if(n->ullman >= UINF && res->ullman >= UINF) fatal("sgen UINF"); - } if(w < 0) fatal("sgen copy %d", w); + if(w == 0) { + // evaluate side effects only. + tempname(&tdst, types[tptr]); + agen(res, &tdst); + agen(n, &tdst); + return; + } + // offset on the stack osrc = stkof(n); odst = stkof(res); diff --git a/src/cmd/8g/cgen64.c b/src/cmd/8g/cgen64.c index ba99cec74..8e568a0f9 100644 --- a/src/cmd/8g/cgen64.c +++ b/src/cmd/8g/cgen64.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "gg.h" /* diff --git a/src/cmd/8g/galign.c b/src/cmd/8g/galign.c index 7734603c4..4526a2efb 100644 --- a/src/cmd/8g/galign.c +++ b/src/cmd/8g/galign.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "gg.h" int thechar = '8'; diff --git a/src/cmd/8g/gg.h b/src/cmd/8g/gg.h index 9f7a66a29..e23ee9e27 100644 --- a/src/cmd/8g/gg.h +++ b/src/cmd/8g/gg.h @@ -2,16 +2,13 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include -#include - -#include "../gc/go.h" -#include "../8l/8.out.h" - #ifndef EXTERN #define EXTERN extern #endif +#include "../gc/go.h" +#include "../8l/8.out.h" + typedef struct Addr Addr; struct Addr @@ -54,15 +51,12 @@ enum Fpop2 = 1<<2, }; -EXTERN Biobuf* bout; EXTERN int32 dynloc; EXTERN uchar reg[D_NONE]; EXTERN int32 pcloc; // instruction counter EXTERN Strlit emptystring; extern char* anames[]; -EXTERN Hist* hist; EXTERN Prog zprog; -EXTERN Node* curfn; EXTERN Node* newproc; EXTERN Node* deferproc; EXTERN Node* deferreturn; @@ -167,12 +161,6 @@ void complexmove(Node*, Node*); void complexgen(Node*, Node*); void complexbool(int, Node*, Node*, int, Prog*); -/* - * gobj.c - */ -void data(void); -void text(void); - /* * list.c */ @@ -185,3 +173,5 @@ void listinit(void); void zaddr(Biobuf*, Addr*, int, int); +#pragma varargck type "D" Addr* +#pragma varargck type "lD" Addr* diff --git a/src/cmd/8g/ggen.c b/src/cmd/8g/ggen.c index 108c493aa..c4f282368 100644 --- a/src/cmd/8g/ggen.c +++ b/src/cmd/8g/ggen.c @@ -4,6 +4,8 @@ #undef EXTERN #define EXTERN +#include +#include #include "gg.h" #include "opt.h" @@ -429,7 +431,6 @@ hard: if(nr->ullman >= nl->ullman || nl->addable) { mgen(nr, &n2, N); nr = &n2; - nr = &n2; } else { tempname(&n2, nr->type); cgen(nr, &n2); @@ -483,8 +484,8 @@ void dodiv(int op, Node *nl, Node *nr, Node *res, Node *ax, Node *dx) { int check; - Node n1, t1, t2, n4, nz; - Type *t; + Node n1, t1, t2, t3, t4, n4, nz; + Type *t, *t0; Prog *p1, *p2, *p3; // Have to be careful about handling @@ -496,6 +497,7 @@ dodiv(int op, Node *nl, Node *nr, Node *res, Node *ax, Node *dx) // For int32 and int64, use explicit test. // Could use int64 hw for int32. t = nl->type; + t0 = t; check = 0; if(issigned[t->etype]) { check = 1; @@ -514,8 +516,18 @@ dodiv(int op, Node *nl, Node *nr, Node *res, Node *ax, Node *dx) tempname(&t1, t); tempname(&t2, t); - cgen(nl, &t1); - cgen(nr, &t2); + if(t0 != t) { + tempname(&t3, t0); + tempname(&t4, t0); + cgen(nl, &t3); + cgen(nr, &t4); + // Convert. + gmove(&t3, &t1); + gmove(&t4, &t2); + } else { + cgen(nl, &t1); + cgen(nr, &t2); + } if(!samereg(ax, res) && !samereg(dx, res)) regalloc(&n1, t, res); diff --git a/src/cmd/8g/gobj.c b/src/cmd/8g/gobj.c index 31c42a3f2..7025a536e 100644 --- a/src/cmd/8g/gobj.c +++ b/src/cmd/8g/gobj.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "gg.h" void @@ -276,54 +278,6 @@ dumpfuncs(void) } } -/* deferred DATA output */ -static Prog *strdat; -static Prog *estrdat; -static int gflag; -static Prog *savepc; - -void -data(void) -{ - gflag = debug['g']; - debug['g'] = 0; - - if(estrdat == nil) { - strdat = mal(sizeof(*pc)); - clearp(strdat); - estrdat = strdat; - } - if(savepc) - fatal("data phase error"); - savepc = pc; - pc = estrdat; -} - -void -text(void) -{ - if(!savepc) - fatal("text phase error"); - debug['g'] = gflag; - estrdat = pc; - pc = savepc; - savepc = nil; -} - -void -dumpdata(void) -{ - Prog *p; - - if(estrdat == nil) - return; - *pc = *strdat; - if(gflag) - for(p=pc; p!=estrdat; p=p->link) - print("%P\n", p); - pc = estrdat; -} - int dsname(Sym *s, int off, char *t, int n) { @@ -380,6 +334,17 @@ gdata(Node *nam, Node *nr, int wid) Prog *p; vlong v; + if(nr->op == OLITERAL) { + switch(nr->val.ctype) { + case CTCPLX: + gdatacomplex(nam, nr->val.u.cval); + return; + case CTSTR: + gdatastring(nam, nr->val.u.sval); + return; + } + } + if(wid == 8 && is64(nr->type)) { v = mpgetfix(nr->val.u.xval); p = gins(ADATA, nam, nodintconst(v)); @@ -547,6 +512,8 @@ genembedtramp(Type *rcvr, Type *method, Sym *newnam, int iface) Prog *p; Type *f; + USED(iface); + e = method->sym; for(d=0; d +#include #include "gg.h" // TODO(rsc): Can make this bigger if we move @@ -48,6 +50,10 @@ clearp(Prog *p) pcloc++; } +static int ddumped; +static Prog *dfirst; +static Prog *dpc; + /* * generate and return proc with p->as = as, * linked into program. pc is next instruction. @@ -57,10 +63,22 @@ prog(int as) { Prog *p; - p = pc; - pc = mal(sizeof(*pc)); - - clearp(pc); + if(as == ADATA || as == AGLOBL) { + if(ddumped) + fatal("already dumped data"); + if(dpc == nil) { + dpc = mal(sizeof(*dpc)); + dfirst = dpc; + } + p = dpc; + dpc = mal(sizeof(*dpc)); + p->link = dpc; + } else { + p = pc; + pc = mal(sizeof(*pc)); + clearp(pc); + p->link = pc; + } if(lineno == 0) { if(debug['K']) @@ -69,10 +87,21 @@ prog(int as) p->as = as; p->lineno = lineno; - p->link = pc; return p; } +void +dumpdata(void) +{ + ddumped = 1; + if(dfirst == nil) + return; + newplist(); + *pc = *dfirst; + pc = dpc; + clearp(pc); +} + /* * generate a branch. * t is ignored. @@ -82,6 +111,7 @@ gbranch(int as, Type *t) { Prog *p; + USED(t); p = prog(as); p->to.type = D_BRANCH; p->to.branch = P; @@ -822,7 +852,7 @@ regalloc(Node *n, Type *t, Node *o) fprint(2, "registers allocated at\n"); for(i=D_AX; i<=D_DI; i++) - fprint(2, "\t%R\t%#ux\n", i, regpc[i]); + fprint(2, "\t%R\t%#lux\n", i, regpc[i]); yyerror("out of fixed registers"); goto err; @@ -832,7 +862,6 @@ regalloc(Node *n, Type *t, Node *o) goto out; } yyerror("regalloc: unknown type %T", t); - i = 0; err: nodreg(n, t, 0); @@ -842,7 +871,7 @@ out: if (i == D_SP) print("alloc SP\n"); if(reg[i] == 0) { - regpc[i] = (ulong)__builtin_return_address(0); + regpc[i] = (ulong)getcallerpc(&n); if(i == D_AX || i == D_CX || i == D_DX || i == D_SP) { dump("regalloc-o", o); fatal("regalloc %R", i); @@ -1809,7 +1838,6 @@ naddr(Node *n, Addr *a, int canemitcode) a->width = n->type->width; a->gotype = ngotype(n); } - a->pun = n->pun; a->offset = n->xoffset; a->sym = n->sym; if(a->sym == S) @@ -1955,5 +1983,9 @@ sudoclean(void) int sudoaddable(int as, Node *n, Addr *a) { + USED(as); + USED(n); + USED(a); + return 0; } diff --git a/src/cmd/8g/list.c b/src/cmd/8g/list.c index edb1ece84..88d3d5f7e 100644 --- a/src/cmd/8g/list.c +++ b/src/cmd/8g/list.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "gg.h" static int sconsize; @@ -128,7 +130,7 @@ Dconv(Fmt *fp) if(fp->flags & FmtLong) { d1 = a->offset; d2 = a->offset2; - snprint(str, sizeof(str), "$%ud-%ud", (ulong)d1, (ulong)d2); + snprint(str, sizeof(str), "$%lud-%lud", (ulong)d1, (ulong)d2); break; } snprint(str, sizeof(str), "$%d", a->offset); diff --git a/src/cmd/8g/opt.h b/src/cmd/8g/opt.h index 8f31dec3b..ed6eb15ab 100644 --- a/src/cmd/8g/opt.h +++ b/src/cmd/8g/opt.h @@ -162,3 +162,5 @@ int32 RtoB(int); int32 FtoB(int); int BtoR(int32); int BtoF(int32); + +#pragma varargck type "D" Adr* diff --git a/src/cmd/8g/peep.c b/src/cmd/8g/peep.c index 5ad29e1b2..e0e44a5ef 100644 --- a/src/cmd/8g/peep.c +++ b/src/cmd/8g/peep.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "gg.h" #include "opt.h" @@ -882,7 +884,6 @@ loop: if(p->from.dval == p0->from.dval) if(p->from.index == p0->from.index) { excise(r); - t++; goto loop; } break; diff --git a/src/cmd/8g/reg.c b/src/cmd/8g/reg.c index 2b878f62a..de5fd87ac 100644 --- a/src/cmd/8g/reg.c +++ b/src/cmd/8g/reg.c @@ -28,9 +28,9 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "gg.h" -#undef EXTERN -#define EXTERN #include "opt.h" #define NREGVAR 8 @@ -98,19 +98,19 @@ setaddrs(Bits bit) { int i, n; Var *v; - Sym *s; + Node *node; while(bany(&bit)) { // convert each bit to a variable i = bnum(bit); - s = var[i].sym; + node = var[i].node; n = var[i].name; bit.b[i/32] &= ~(1L<<(i%32)); // disable all pieces of that variable for(i=0; isym == s && v->name == n) + if(v->node == node && v->name == n) v->addr = 2; } } @@ -155,7 +155,7 @@ regopt(Prog *firstp) nvar = NREGVAR; memset(var, 0, NREGVAR*sizeof var[0]); for(i=0; ito; - a->sym = v->sym; a->offset = v->offset; a->etype = v->etype; a->type = v->name; a->gotype = v->gotype; a->node = v->node; + a->sym = v->node->sym; // need to clean this up with wptr and // some of the defaults @@ -810,7 +810,7 @@ mkvar(Reg *r, Adr *a) int i, t, n, et, z, w, flag, regu; int32 o; Bits bit; - Sym *s; + Node *node; /* * mark registers used @@ -847,10 +847,11 @@ mkvar(Reg *r, Adr *a) break; } - s = a->sym; - if(s == S) + node = a->node; + if(node == N || node->op != ONAME || node->orig != N) goto none; - if(s->name[0] == '.') + node = node->orig; + if(node->sym->name[0] == '.') goto none; et = a->etype; o = a->offset; @@ -859,7 +860,7 @@ mkvar(Reg *r, Adr *a) flag = 0; for(i=0; isym == s && v->name == n) { + if(v->node == node && v->name == n) { if(v->offset == o) if(v->etype == et) if(v->width == w) @@ -868,7 +869,7 @@ mkvar(Reg *r, Adr *a) // if they overlap, disable both if(overlap(v->offset, v->width, o, w)) { if(debug['R']) - print("disable %s\n", v->sym->name); + print("disable %s\n", node->sym->name); v->addr = 1; flag = 1; } @@ -882,7 +883,7 @@ mkvar(Reg *r, Adr *a) } if(nvar >= NVAR) { - if(debug['w'] > 1 && s) + if(debug['w'] > 1 && node != N) fatal("variable not optimized: %D", a); goto none; } @@ -890,17 +891,16 @@ mkvar(Reg *r, Adr *a) i = nvar; nvar++; v = var+i; - v->sym = s; v->offset = o; v->name = n; v->gotype = a->gotype; v->etype = et; v->width = w; v->addr = flag; // funny punning - v->node = a->node; + v->node = node; if(debug['R']) - print("bit=%2d et=%2d w=%d+%d %S %D flag=%d\n", i, et, o, w, s, a, v->addr); + print("bit=%2d et=%2d w=%d+%d %#N %D flag=%d\n", i, et, o, w, node, a, v->addr); ostats.nvar++; bit = blsh(i); diff --git a/src/cmd/8l/8.out.h b/src/cmd/8l/8.out.h index 9a8483aaf..5e4b73ad0 100644 --- a/src/cmd/8l/8.out.h +++ b/src/cmd/8l/8.out.h @@ -115,7 +115,8 @@ enum as AIRETW, AJCC, AJCS, - AJCXZ, + AJCXZL, + AJCXZW, AJEQ, AJGE, AJGT, diff --git a/src/cmd/8l/asm.c b/src/cmd/8l/asm.c index 22abd8049..518951fde 100644 --- a/src/cmd/8l/asm.c +++ b/src/cmd/8l/asm.c @@ -41,6 +41,7 @@ char linuxdynld[] = "/lib/ld-linux.so.2"; char freebsddynld[] = "/usr/libexec/ld-elf.so.1"; +char openbsddynld[] = "/usr/libexec/ld.so"; int32 entryvalue(void) @@ -956,6 +957,9 @@ asmb(void) case Hfreebsd: interpreter = freebsddynld; break; + case Hopenbsd: + interpreter = openbsddynld; + break; } } elfinterp(sh, startva, interpreter); @@ -1122,7 +1126,10 @@ asmb(void) eh->ident[EI_VERSION] = EV_CURRENT; switch(HEADTYPE) { case Hfreebsd: - eh->ident[EI_OSABI] = 9; + eh->ident[EI_OSABI] = ELFOSABI_FREEBSD; + break; + case Hopenbsd: + eh->ident[EI_OSABI] = ELFOSABI_OPENBSD; break; } diff --git a/src/cmd/8l/doc.go b/src/cmd/8l/doc.go index b70888907..de877bb24 100644 --- a/src/cmd/8l/doc.go +++ b/src/cmd/8l/doc.go @@ -33,6 +33,8 @@ Options new in this version: Write Linux ELF binaries (default when $GOOS is linux) -Hfreebsd Write FreeBSD ELF binaries (default when $GOOS is freebsd) +-Hopenbsd + Write OpenBSD ELF binaries (default when $GOOS is openbsd) -Hwindows Write Windows PE32 binaries (default when $GOOS is windows) -I interpreter diff --git a/src/cmd/8l/l.h b/src/cmd/8l/l.h index 4ee0db967..a721f384b 100644 --- a/src/cmd/8l/l.h +++ b/src/cmd/8l/l.h @@ -209,6 +209,7 @@ enum Zbr, Zcall, Zcallcon, + Zcallind, Zib_, Zib_rp, Zibo_m, diff --git a/src/cmd/8l/obj.c b/src/cmd/8l/obj.c index a8e1c34a5..297b5bed5 100644 --- a/src/cmd/8l/obj.c +++ b/src/cmd/8l/obj.c @@ -55,6 +55,7 @@ Header headers[] = { "darwin", Hdarwin, "linux", Hlinux, "freebsd", Hfreebsd, + "openbsd", Hopenbsd, "windows", Hwindows, "windowsgui", Hwindows, 0, 0 @@ -69,6 +70,7 @@ Header headers[] = { * -Hdarwin -Tx -Rx is Apple Mach-O * -Hlinux -Tx -Rx is Linux ELF32 * -Hfreebsd -Tx -Rx is FreeBSD ELF32 + * -Hopenbsd -Tx -Rx is OpenBSD ELF32 * -Hwindows -Tx -Rx is MS Windows PE32 */ @@ -223,6 +225,7 @@ main(int argc, char *argv[]) break; case Hlinux: /* elf32 executable */ case Hfreebsd: + case Hopenbsd: /* * ELF uses TLS offsets negative from %gs. * Translate 0(GS) and 4(GS) into -8(GS) and -4(GS). diff --git a/src/cmd/8l/optab.c b/src/cmd/8l/optab.c index f5c195d75..69602d704 100644 --- a/src/cmd/8l/optab.c +++ b/src/cmd/8l/optab.c @@ -260,6 +260,7 @@ uchar yloop[] = uchar ycall[] = { Ynone, Yml, Zo_m, 2, + Ynone, Ycol, Zcallind, 2, Ynone, Ybr, Zcall, 0, Ynone, Yi32, Zcallcon, 1, 0 @@ -383,7 +384,7 @@ Optab optab[] = { ABTSL, yml_rl, Pm, 0xab }, { ABTSW, yml_rl, Pq, 0xab }, { ABYTE, ybyte, Px, 1 }, - { ACALL, ycall, Px, 0xff,(02),0xe8 }, + { ACALL, ycall, Px, 0xff,(02),0xff,(0x15),0xe8 }, { ACLC, ynone, Px, 0xf8 }, { ACLD, ynone, Px, 0xfc }, { ACLI, ynone, Px, 0xfa }, @@ -430,7 +431,8 @@ Optab optab[] = { AIRETW, ynone, Pe, 0xcf }, { AJCC, yjcond, Px, 0x73,0x83,(00) }, { AJCS, yjcond, Px, 0x72,0x82 }, - { AJCXZ, yloop, Px, 0xe3 }, + { AJCXZL, yloop, Px, 0xe3 }, + { AJCXZW, yloop, Px, 0xe3 }, { AJEQ, yjcond, Px, 0x74,0x84 }, { AJGE, yjcond, Px, 0x7d,0x8d }, { AJGT, yjcond, Px, 0x7f,0x8f }, diff --git a/src/cmd/8l/pass.c b/src/cmd/8l/pass.c index 2e0990c5a..54ea965da 100644 --- a/src/cmd/8l/pass.c +++ b/src/cmd/8l/pass.c @@ -307,9 +307,12 @@ patch(void) p->from.offset = 0; } } - if(p->as == ACALL || (p->as == AJMP && p->to.type != D_BRANCH)) { + if((p->as == ACALL && p->to.type != D_BRANCH) || (p->as == AJMP && p->to.type != D_BRANCH)) { s = p->to.sym; - if(s) { + if(p->to.type == D_INDIR+D_ADDR) { + /* skip check if this is an indirect call (CALL *symbol(SB)) */ + continue; + } else if(s) { if(debug['c']) Bprint(&bso, "%s calls %s\n", TNAME, s->name); if((s->type&~SSUB) != STEXT) { diff --git a/src/cmd/8l/span.c b/src/cmd/8l/span.c index a4cba1257..81c1d37eb 100644 --- a/src/cmd/8l/span.c +++ b/src/cmd/8l/span.c @@ -83,7 +83,10 @@ span1(Sym *s) loop++; q->back ^= 2; } - s->p[q->pc+1] = v; + if(q->as == AJCXZW) + s->p[q->pc+2] = v; + else + s->p[q->pc+1] = v; } else { bp = s->p + q->pc + q->mark - 4; *bp++ = v; @@ -282,6 +285,8 @@ oclass(Adr *a) } return Yxxx; } + //if(a->type == D_INDIR+D_ADDR) + // print("*Ycol\n"); return Ycol; } return Ym; @@ -1056,9 +1061,10 @@ found: case Zbr: case Zjmp: + case Zloop: q = p->pcond; if(q == nil) { - diag("jmp/branch without target"); + diag("jmp/branch/loop without target"); errorexit(); } if(q->as == ATEXT) { @@ -1084,8 +1090,12 @@ found: if(p->back & 1) { v = q->pc - (p->pc + 2); if(v >= -128) { + if(p->as == AJCXZW) + *andptr++ = 0x67; *andptr++ = op; *andptr++ = v; + } else if(t[2] == Zloop) { + diag("loop too far: %P", p); } else { v -= 5-2; if(t[2] == Zbr) { @@ -1105,8 +1115,12 @@ found: p->forwd = q->comefrom; q->comefrom = p; if(p->back & 2) { // short + if(p->as == AJCXZW) + *andptr++ = 0x67; *andptr++ = op; *andptr++ = 0; + } else if(t[2] == Zloop) { + diag("loop too far: %P", p); } else { if(t[2] == Zbr) *andptr++ = 0x0f; @@ -1131,18 +1145,17 @@ found: r->add = p->to.offset; put4(0); break; - - case Zloop: - q = p->pcond; - if(q == nil) { - diag("loop without target"); - errorexit(); - } - v = q->pc - p->pc - 2; - if(v < -128 && v > 127) - diag("loop too far: %P", p); + + case Zcallind: *andptr++ = op; - *andptr++ = v; + *andptr++ = o->op[z+1]; + r = addrel(cursym); + r->off = p->pc + andptr - and; + r->type = D_ADDR; + r->siz = 4; + r->add = p->to.offset; + r->sym = p->to.sym; + put4(0); break; case Zbyte: diff --git a/src/cmd/cgo/doc.go b/src/cmd/cgo/doc.go index 63413825a..7faece81c 100644 --- a/src/cmd/cgo/doc.go +++ b/src/cmd/cgo/doc.go @@ -73,6 +73,9 @@ A few special functions convert between Go and C types by making copies of the data. In pseudo-Go definitions: // Go string to C string + // The C string is allocated in the C heap using malloc. + // It is the caller's responsibility to arrange for it to be + // freed, such as by calling C.free. func C.CString(string) *C.char // C string to Go string diff --git a/src/cmd/cov/Makefile b/src/cmd/cov/Makefile index 95dba9c60..62836fcac 100644 --- a/src/cmd/cov/Makefile +++ b/src/cmd/cov/Makefile @@ -29,6 +29,7 @@ endif install: install-$(NAME) install-linux: install-default install-freebsd: install-default +install-openbsd: install-default install-windows: install-default # on Darwin, have to install and setgid; see $GOROOT/src/sudo.bash diff --git a/src/cmd/cov/main.c b/src/cmd/cov/main.c index 5ff22c00a..ecbabf371 100644 --- a/src/cmd/cov/main.c +++ b/src/cmd/cov/main.c @@ -337,6 +337,9 @@ cover(void) uvlong rgetzero(Map *map, char *reg) { + USED(map); + USED(reg); + return 0; } diff --git a/src/cmd/cov/tree.c b/src/cmd/cov/tree.c index 116772e42..905bb7d97 100644 --- a/src/cmd/cov/tree.c +++ b/src/cmd/cov/tree.c @@ -2,7 +2,7 @@ /* Copyright (c) 2003-2007 Russ Cox, Tom Bergan, Austin Clements, - Massachusetts Institute of Technology + Massachusetts Institute of Technology Portions Copyright (c) 2009 The Go Authors. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining @@ -35,9 +35,6 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include #include "tree.h" -#define TreeNode TreeNode -#define Tree Tree - enum { Red = 0, diff --git a/src/cmd/gc/Makefile b/src/cmd/gc/Makefile index 0af7659e4..f7e305178 100644 --- a/src/cmd/gc/Makefile +++ b/src/cmd/gc/Makefile @@ -22,6 +22,7 @@ OFILES=\ closure.$O\ const.$O\ dcl.$O\ + esc.$O\ export.$O\ gen.$O\ init.$O\ diff --git a/src/cmd/gc/align.c b/src/cmd/gc/align.c index 14c1c4a8d..f316c19e0 100644 --- a/src/cmd/gc/align.c +++ b/src/cmd/gc/align.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" /* diff --git a/src/cmd/gc/bits.c b/src/cmd/gc/bits.c index 7188ac411..f3b031cc3 100644 --- a/src/cmd/gc/bits.c +++ b/src/cmd/gc/bits.c @@ -28,6 +28,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +#include +#include #include "go.h" /* @@ -148,12 +150,12 @@ Qconv(Fmt *fp) first = 0; else fmtprint(fp, " "); - if(var[i].sym == S) + if(var[i].node == N || var[i].node->sym == S) fmtprint(fp, "$%lld", var[i].offset); else { - fmtprint(fp, var[i].sym->name); + fmtprint(fp, var[i].node->sym->name); if(var[i].offset != 0) - fmtprint(fp, "%+d", var[i].offset); + fmtprint(fp, "%+lld", (vlong)var[i].offset); } bits.b[i/32] &= ~(1L << (i%32)); } diff --git a/src/cmd/gc/closure.c b/src/cmd/gc/closure.c index 1261eefb7..d29e8cbc2 100644 --- a/src/cmd/gc/closure.c +++ b/src/cmd/gc/closure.c @@ -6,6 +6,8 @@ * function literals aka closures */ +#include +#include #include "go.h" void @@ -57,7 +59,6 @@ closurebody(NodeList *body) body = list1(nod(OEMPTY, N, N)); func = curfn; - l = func->dcl; func->nbody = body; funcbody(func); @@ -129,6 +130,8 @@ makeclosure(Node *func, NodeList **init, int nowrap) static int closgen; char *p; + USED(init); + /* * wrap body in external function * with extra closure parameters. diff --git a/src/cmd/gc/const.c b/src/cmd/gc/const.c index 36a64cb97..135a8102e 100644 --- a/src/cmd/gc/const.c +++ b/src/cmd/gc/const.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" #define TUP(x,y) (((x)<<16)|(y)) @@ -1099,7 +1101,7 @@ cmpslit(Node *l, Node *r) int smallintconst(Node *n) { - if(n->op == OLITERAL && n->type != T) + if(n->op == OLITERAL && n->val.ctype == CTINT && n->type != T) switch(simtype[n->type->etype]) { case TINT8: case TUINT8: @@ -1110,6 +1112,7 @@ smallintconst(Node *n) case TBOOL: case TPTR32: return 1; + case TIDEAL: case TINT64: case TUINT64: if(mpcmpfixfix(n->val.u.xval, minintval[TINT32]) < 0 diff --git a/src/cmd/gc/cplx.c b/src/cmd/gc/cplx.c index 890cf7f10..5bc8d64aa 100644 --- a/src/cmd/gc/cplx.c +++ b/src/cmd/gc/cplx.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "gg.h" static void subnode(Node *nr, Node *ni, Node *nc); diff --git a/src/cmd/gc/dcl.c b/src/cmd/gc/dcl.c index 5bfeeb97a..d8b89b4f3 100644 --- a/src/cmd/gc/dcl.c +++ b/src/cmd/gc/dcl.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" #include "y.tab.h" @@ -115,6 +117,8 @@ dumpdcl(char *st) Sym *s, *d; int i; + USED(st); + i = 0; for(d=dclstack; d!=S; d=d->link) { i++; @@ -188,7 +192,7 @@ declare(Node *n, int ctxt) n->curfn = curfn; } if(ctxt == PAUTO) - n->xoffset = BADWIDTH; + n->xoffset = 0; if(s->block == block) redeclare(s, "in this block"); @@ -726,7 +730,6 @@ stotype(NodeList *l, int et, Type **t, int funarg) for(; l; l=l->next) { n = l->n; lineno = n->lineno; - note = nil; if(n->op != ODCLFIELD) fatal("stotype: oops %N\n", n); @@ -820,6 +823,10 @@ stotype(NodeList *l, int et, Type **t, int funarg) f->width = BADWIDTH; f->isddd = n->isddd; + // esc.c needs to find f given a PPARAM to add the tag. + if(funarg && n->left && n->left->class == PPARAM) + n->left->paramfld = f; + if(left != N && left->op == ONAME) { f->nname = left; f->embedded = n->embedded; @@ -1133,8 +1140,6 @@ addmethod(Sym *sf, Type *t, int local) Type *f, *d, *pa; Node *n; - pa = nil; - // get field sym if(sf == S) fatal("no method symbol"); diff --git a/src/cmd/gc/doc.go b/src/cmd/gc/doc.go index 3fe7fafdd..5bb5e0e14 100644 --- a/src/cmd/gc/doc.go +++ b/src/cmd/gc/doc.go @@ -35,12 +35,13 @@ Flags: output file, default file.6 for 6g, etc. -e normally the compiler quits after 10 errors; -e prints all errors + -p path + assume that path is the eventual import path for this code, + and diagnose any attempt to import a package that depends on it. -L show entire file path when printing line numbers in errors -I dir1 -I dir2 add dir1 and dir2 to the list of paths to check for imported packages - -N - disable optimization -S write assembly language text to standard output -u diff --git a/src/cmd/gc/esc.c b/src/cmd/gc/esc.c new file mode 100644 index 000000000..cd1f9770e --- /dev/null +++ b/src/cmd/gc/esc.c @@ -0,0 +1,723 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// Escape analysis. +// +// First escfunc, esc and escassign recurse over the ast of each +// function to dig out flow(dst,src) edges between any +// pointer-containing nodes and store them in dst->escflowsrc. For +// variables assigned to a variable in an outer scope or used as a +// return value, they store a flow(theSink, src) edge to a fake node +// 'the Sink'. For variables referenced in closures, an edge +// flow(closure, &var) is recorded and the flow of a closure itself to +// an outer scope is tracked the same way as other variables. +// +// Then escflood walks the graph starting at theSink and tags all +// variables of it can reach an & node as escaping and all function +// parameters it can reach as leaking. +// +// If a value's address is taken but the address does not escape, +// then the value can stay on the stack. If the value new(T) does +// not escape, then new(T) can be rewritten into a stack allocation. +// The same is true of slice literals. +// +// If escape analysis is disabled (-s), this code is not used. +// Instead, the compiler assumes that any value whose address +// is taken without being immediately dereferenced +// needs to be moved to the heap, and new(T) and slice +// literals are always real allocations. + +#include +#include +#include "go.h" + +static void escfunc(Node *func); +static void esclist(NodeList *l); +static void esc(Node *n); +static void escassign(Node *dst, Node *src); +static void esccall(Node*); +static void escflows(Node *dst, Node *src); +static void escflood(Node *dst); +static void escwalk(int level, Node *dst, Node *src); +static void esctag(Node *func); + +// Fake node that all +// - return values and output variables +// - parameters on imported functions not marked 'safe' +// - assignments to global variables +// flow to. +static Node theSink; + +static NodeList* dsts; // all dst nodes +static int loopdepth; // for detecting nested loop scopes +static int pdepth; // for debug printing in recursions. +static Strlit* safetag; // gets slapped on safe parameters' field types for export +static int dstcount, edgecount; // diagnostic +static NodeList* noesc; // list of possible non-escaping nodes, for printing + +void +escapes(void) +{ + NodeList *l; + + theSink.op = ONAME; + theSink.class = PEXTERN; + theSink.sym = lookup(".sink"); + theSink.escloopdepth = -1; + + safetag = strlit("noescape"); + + // flow-analyze top level functions + for(l=xtop; l; l=l->next) + if(l->n->op == ODCLFUNC || l->n->op == OCLOSURE) + escfunc(l->n); + + // print("escapes: %d dsts, %d edges\n", dstcount, edgecount); + + // visit the updstream of each dst, mark address nodes with + // addrescapes, mark parameters unsafe + for(l = dsts; l; l=l->next) + escflood(l->n); + + // for all top level functions, tag the typenodes corresponding to the param nodes + for(l=xtop; l; l=l->next) + if(l->n->op == ODCLFUNC) + esctag(l->n); + + if(debug['m']) { + for(l=noesc; l; l=l->next) + if(l->n->esc == EscNone) + warnl(l->n->lineno, "%S %#N does not escape", + (l->n->curfn && l->n->curfn->nname) ? l->n->curfn->nname->sym : S, + l->n); + } +} + + +static void +escfunc(Node *func) +{ + Node *savefn, *n; + NodeList *ll; + int saveld; + + saveld = loopdepth; + loopdepth = 1; + savefn = curfn; + curfn = func; + + for(ll=curfn->dcl; ll; ll=ll->next) { + if(ll->n->op != ONAME) + continue; + switch (ll->n->class) { + case PPARAMOUT: + // output parameters flow to the sink + escflows(&theSink, ll->n); + ll->n->escloopdepth = loopdepth; + break; + case PPARAM: + if(ll->n->type && !haspointers(ll->n->type)) + break; + ll->n->esc = EscNone; // prime for escflood later + noesc = list(noesc, ll->n); + ll->n->escloopdepth = loopdepth; + break; + } + } + + // walk will take the address of cvar->closure later and assign it to cvar. + // handle that here by linking a fake oaddr node directly to the closure. + for(ll=curfn->cvars; ll; ll=ll->next) { + if(ll->n->op == OXXX) // see dcl.c:398 + continue; + + n = nod(OADDR, ll->n->closure, N); + n->lineno = ll->n->lineno; + typecheck(&n, Erv); + escassign(curfn, n); + } + + esclist(curfn->nbody); + curfn = savefn; + loopdepth = saveld; +} + +static void +esclist(NodeList *l) +{ + for(; l; l=l->next) + esc(l->n); +} + +static void +esc(Node *n) +{ + int lno; + NodeList *ll, *lr; + + if(n == N) + return; + + lno = setlineno(n); + + if(n->op == OFOR || n->op == ORANGE) + loopdepth++; + + esc(n->left); + esc(n->right); + esc(n->ntest); + esc(n->nincr); + esclist(n->ninit); + esclist(n->nbody); + esclist(n->nelse); + esclist(n->list); + esclist(n->rlist); + + if(n->op == OFOR || n->op == ORANGE) + loopdepth--; + + if(debug['m'] > 1) + print("%L:[%d] %#S esc: %#N\n", lineno, loopdepth, + (curfn && curfn->nname) ? curfn->nname->sym : S, n); + + switch(n->op) { + case ODCL: + // Record loop depth at declaration. + if(n->left) + n->left->escloopdepth = loopdepth; + break; + + case OLABEL: // TODO: new loop/scope only if there are backjumps to it. + loopdepth++; + break; + + case ORANGE: + // Everything but fixed array is a dereference. + if(isfixedarray(n->type) && n->list->next) + escassign(n->list->next->n, n->right); + break; + + case OSWITCH: + if(n->ntest && n->ntest->op == OTYPESW) { + for(ll=n->list; ll; ll=ll->next) { // cases + // ntest->right is the argument of the .(type), + // ll->n->nname is the variable per case + escassign(ll->n->nname, n->ntest->right); + } + } + break; + + case OAS: + case OASOP: + escassign(n->left, n->right); + break; + + case OAS2: // x,y = a,b + if(count(n->list) == count(n->rlist)) + for(ll=n->list, lr=n->rlist; ll; ll=ll->next, lr=lr->next) + escassign(ll->n, lr->n); + break; + + case OAS2RECV: // v, ok = <-ch + case OAS2MAPR: // v, ok = m[k] + case OAS2DOTTYPE: // v, ok = x.(type) + case OAS2MAPW: // m[k] = x, ok + escassign(n->list->n, n->rlist->n); + break; + + case OSEND: // ch <- x + escassign(&theSink, n->right); + break; + + case ODEFER: + if(loopdepth == 1) // top level + break; + // arguments leak out of scope + // TODO: leak to a dummy node instead + // fallthrough + case OPROC: + // go f(x) - f and x escape + escassign(&theSink, n->left->left); + for(ll=n->left->list; ll; ll=ll->next) + escassign(&theSink, ll->n); + break; + + case ORETURN: + for(ll=n->list; ll; ll=ll->next) + escassign(&theSink, ll->n); + break; + + case OPANIC: + // Argument could leak through recover. + escassign(&theSink, n->left); + break; + + case OAPPEND: + if(!n->isddd) + for(ll=n->list->next; ll; ll=ll->next) + escassign(&theSink, ll->n); // lose track of assign to dereference + break; + + case OCALLMETH: + case OCALLFUNC: + case OCALLINTER: + esccall(n); + break; + + case OCONV: + case OCONVNOP: + case OCONVIFACE: + escassign(n, n->left); + break; + + case OARRAYLIT: + if(isslice(n->type)) { + n->esc = EscNone; // until proven otherwise + noesc = list(noesc, n); + n->escloopdepth = loopdepth; + // Values make it to memory, lose track. + for(ll=n->list; ll; ll=ll->next) + escassign(&theSink, ll->n->right); + } else { + // Link values to array. + for(ll=n->list; ll; ll=ll->next) + escassign(n, ll->n->right); + } + break; + + case OSTRUCTLIT: + // Link values to struct. + for(ll=n->list; ll; ll=ll->next) + escassign(n, ll->n->right); + break; + + case OMAPLIT: + n->esc = EscNone; // until proven otherwise + noesc = list(noesc, n); + n->escloopdepth = loopdepth; + // Keys and values make it to memory, lose track. + for(ll=n->list; ll; ll=ll->next) { + escassign(&theSink, ll->n->left); + escassign(&theSink, ll->n->right); + } + break; + + case OADDR: + case OCLOSURE: + case OMAKECHAN: + case OMAKEMAP: + case OMAKESLICE: + case ONEW: + n->escloopdepth = loopdepth; + n->esc = EscNone; // until proven otherwise + noesc = list(noesc, n); + break; + } + + lineno = lno; +} + +// Assert that expr somehow gets assigned to dst, if non nil. for +// dst==nil, any name node expr still must be marked as being +// evaluated in curfn. For expr==nil, dst must still be examined for +// evaluations inside it (e.g *f(x) = y) +static void +escassign(Node *dst, Node *src) +{ + int lno; + + if(isblank(dst) || dst == N || src == N || src->op == ONONAME || src->op == OXXX) + return; + + if(debug['m'] > 1) + print("%L:[%d] %#S escassign: %hN = %hN\n", lineno, loopdepth, + (curfn && curfn->nname) ? curfn->nname->sym : S, dst, src); + + setlineno(dst); + + // Analyze lhs of assignment. + // Replace dst with theSink if we can't track it. + switch(dst->op) { + default: + dump("dst", dst); + fatal("escassign: unexpected dst"); + + case OARRAYLIT: + case OCLOSURE: + case OCONV: + case OCONVIFACE: + case OCONVNOP: + case OMAPLIT: + case OSTRUCTLIT: + break; + + case ONAME: + if(dst->class == PEXTERN) + dst = &theSink; + break; + case ODOT: // treat "dst.x = src" as "dst = src" + escassign(dst->left, src); + return; + case OINDEX: + if(isfixedarray(dst->left->type)) { + escassign(dst->left, src); + return; + } + dst = &theSink; // lose track of dereference + break; + case OIND: + case ODOTPTR: + dst = &theSink; // lose track of dereference + break; + case OINDEXMAP: + // lose track of key and value + escassign(&theSink, dst->right); + dst = &theSink; + break; + } + + lno = setlineno(src); + pdepth++; + + switch(src->op) { + case OADDR: // dst = &x + case OIND: // dst = *x + case ODOTPTR: // dst = (*x).f + case ONAME: + case OPARAM: + case ODDDARG: + case OARRAYLIT: + case OMAPLIT: + case OSTRUCTLIT: + // loopdepth was set in the defining statement or function header + escflows(dst, src); + break; + + case OCONV: + case OCONVIFACE: + case OCONVNOP: + case ODOT: + case ODOTMETH: // treat recv.meth as a value with recv in it, only happens in ODEFER and OPROC + // iface.method already leaks iface in esccall, no need to put in extra ODOTINTER edge here + case ODOTTYPE: + case ODOTTYPE2: + case OSLICE: + case OSLICEARR: + // Conversions, field access, slice all preserve the input value. + escassign(dst, src->left); + break; + + case OAPPEND: + // Append returns first argument. + escassign(dst, src->list->n); + break; + + case OINDEX: + // Index of array preserves input value. + if(isfixedarray(src->left->type)) + escassign(dst, src->left); + break; + + case OMAKECHAN: + case OMAKEMAP: + case OMAKESLICE: + case ONEW: + escflows(dst, src); + break; + + case OCLOSURE: + escflows(dst, src); + escfunc(src); + break; + + case OADD: + case OSUB: + case OOR: + case OXOR: + case OMUL: + case ODIV: + case OMOD: + case OLSH: + case ORSH: + case OAND: + case OANDNOT: + case OPLUS: + case OMINUS: + case OCOM: + // Might be pointer arithmetic, in which case + // the operands flow into the result. + // TODO(rsc): Decide what the story is here. This is unsettling. + escassign(dst, src->left); + escassign(dst, src->right); + break; + + } + + pdepth--; + lineno = lno; +} + + +// This is a bit messier than fortunate, pulled out of escassign's big +// switch for clarity. We either have the paramnodes, which may be +// connected to other things throug flows or we have the parameter type +// nodes, which may be marked 'n(ofloworescape)'. Navigating the ast is slightly +// different for methods vs plain functions and for imported vs +// this-package +static void +esccall(Node *n) +{ + NodeList *ll, *lr; + Node *a, *fn, *src; + Type *t, *fntype; + + fn = N; + switch(n->op) { + default: + fatal("esccall"); + + case OCALLFUNC: + fn = n->left; + fntype = fn->type; + break; + + case OCALLMETH: + fn = n->left->right->sym->def; + if(fn) + fntype = fn->type; + else + fntype = n->left->type; + break; + + case OCALLINTER: + fntype = n->left->type; + break; + } + + ll = n->list; + if(n->list != nil && n->list->next == nil) { + a = n->list->n; + if(a->type->etype == TSTRUCT && a->type->funarg) { + // f(g()). + // Since f's arguments are g's results and + // all function results escape, we're done. + ll = nil; + } + } + + if(fn && fn->op == ONAME && fn->class == PFUNC && fn->defn && fn->defn->nbody && fn->ntype) { + // Local function. Incorporate into flow graph. + + // Receiver. + if(n->op != OCALLFUNC) + escassign(fn->ntype->left->left, n->left->left); + + for(lr=fn->ntype->list; ll && lr; ll=ll->next, lr=lr->next) { + src = ll->n; + if(lr->n->isddd && !n->isddd) { + // Introduce ODDDARG node to represent ... allocation. + src = nod(ODDDARG, N, N); + src->escloopdepth = loopdepth; + src->lineno = n->lineno; + src->esc = EscNone; // until we find otherwise + noesc = list(noesc, src); + n->right = src; + } + if(lr->n->left != N) + escassign(lr->n->left, src); + if(src != ll->n) + break; + } + // "..." arguments are untracked + for(; ll; ll=ll->next) + escassign(&theSink, ll->n); + return; + } + + // Imported function. Use the escape tags. + if(n->op != OCALLFUNC) { + t = getthisx(fntype)->type; + if(!t->note || strcmp(t->note->s, safetag->s) != 0) + escassign(&theSink, n->left->left); + } + for(t=getinargx(fntype)->type; ll; ll=ll->next) { + src = ll->n; + if(t->isddd && !n->isddd) { + // Introduce ODDDARG node to represent ... allocation. + src = nod(ODDDARG, N, N); + src->escloopdepth = loopdepth; + src->lineno = n->lineno; + src->esc = EscNone; // until we find otherwise + noesc = list(noesc, src); + n->right = src; + } + if(!t->note || strcmp(t->note->s, safetag->s) != 0) + escassign(&theSink, src); + if(src != ll->n) + break; + t = t->down; + } + // "..." arguments are untracked + for(; ll; ll=ll->next) + escassign(&theSink, ll->n); +} + +// Store the link src->dst in dst, throwing out some quick wins. +static void +escflows(Node *dst, Node *src) +{ + if(dst == nil || src == nil || dst == src) + return; + + // Don't bother building a graph for scalars. + if(src->type && !haspointers(src->type)) + return; + + if(debug['m']>2) + print("%L::flows:: %hN <- %hN\n", lineno, dst, src); + + if(dst->escflowsrc == nil) { + dsts = list(dsts, dst); + dstcount++; + } + edgecount++; + + dst->escflowsrc = list(dst->escflowsrc, src); +} + +// Whenever we hit a reference node, the level goes up by one, and whenever +// we hit an OADDR, the level goes down by one. as long as we're on a level > 0 +// finding an OADDR just means we're following the upstream of a dereference, +// so this address doesn't leak (yet). +// If level == 0, it means the /value/ of this node can reach the root of this flood. +// so if this node is an OADDR, it's argument should be marked as escaping iff +// it's currfn/loopdepth are different from the flood's root. +// Once an object has been moved to the heap, all of it's upstream should be considered +// escaping to the global scope. +static void +escflood(Node *dst) +{ + NodeList *l; + + switch(dst->op) { + case ONAME: + case OCLOSURE: + break; + default: + return; + } + + if(debug['m']>1) + print("\nescflood:%d: dst %hN scope:%#S[%d]\n", walkgen, dst, + (dst->curfn && dst->curfn->nname) ? dst->curfn->nname->sym : S, + dst->escloopdepth); + + for(l = dst->escflowsrc; l; l=l->next) { + walkgen++; + escwalk(0, dst, l->n); + } +} + +static void +escwalk(int level, Node *dst, Node *src) +{ + NodeList *ll; + int leaks; + + if(src->walkgen == walkgen) + return; + src->walkgen = walkgen; + + if(debug['m']>1) + print("escwalk: level:%d depth:%d %.*s %hN scope:%#S[%d]\n", + level, pdepth, pdepth, "\t\t\t\t\t\t\t\t\t\t", src, + (src->curfn && src->curfn->nname) ? src->curfn->nname->sym : S, src->escloopdepth); + + pdepth++; + + leaks = (level <= 0) && (dst->escloopdepth < src->escloopdepth); + + switch(src->op) { + case ONAME: + if(src->class == PPARAM && leaks && src->esc == EscNone) { + src->esc = EscScope; + if(debug['m']) + warnl(src->lineno, "leaking param: %hN", src); + } + break; + + case OADDR: + if(leaks) { + src->esc = EscHeap; + addrescapes(src->left); + if(debug['m']) + warnl(src->lineno, "%#N escapes to heap", src); + } + escwalk(level-1, dst, src->left); + break; + + case OARRAYLIT: + if(isfixedarray(src->type)) + break; + // fall through + case ODDDARG: + case OMAKECHAN: + case OMAKEMAP: + case OMAKESLICE: + case OMAPLIT: + case ONEW: + case OCLOSURE: + if(leaks) { + src->esc = EscHeap; + if(debug['m']) + warnl(src->lineno, "%#N escapes to heap", src); + } + break; + + case OINDEX: + if(isfixedarray(src->type)) + break; + // fall through + case OSLICE: + case ODOTPTR: + case OINDEXMAP: + case OIND: + escwalk(level+1, dst, src->left); + } + + for(ll=src->escflowsrc; ll; ll=ll->next) + escwalk(level, dst, ll->n); + + pdepth--; +} + +static void +esctag(Node *func) +{ + Node *savefn; + NodeList *ll; + + // External functions must be assumed unsafe. + if(func->nbody == nil) + return; + + savefn = curfn; + curfn = func; + + for(ll=curfn->dcl; ll; ll=ll->next) { + if(ll->n->op != ONAME || ll->n->class != PPARAM) + continue; + + switch (ll->n->esc) { + case EscNone: // not touched by escflood + if(haspointers(ll->n->type)) // don't bother tagging for scalars + ll->n->paramfld->note = safetag; + case EscHeap: // touched by escflood, moved to heap + case EscScope: // touched by escflood, value leaves scope + break; + } + } + + curfn = savefn; +} diff --git a/src/cmd/gc/export.c b/src/cmd/gc/export.c index 014f0c5f0..421afda8b 100644 --- a/src/cmd/gc/export.c +++ b/src/cmd/gc/export.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" #include "y.tab.h" diff --git a/src/cmd/gc/gen.c b/src/cmd/gc/gen.c index cb66921ba..a818dbc19 100644 --- a/src/cmd/gc/gen.c +++ b/src/cmd/gc/gen.c @@ -7,11 +7,13 @@ * mainly statements and control flow. */ +#include +#include #include "go.h" static void cgen_dcl(Node *n); static void cgen_proc(Node *n, int proc); -static void checkgoto(Node*, Node*); +static void checkgoto(Node*, Node*); static Label *labellist; static Label *lastlabel; @@ -26,50 +28,93 @@ sysfunc(char *name) return n; } +/* + * the address of n has been taken and might be used after + * the current function returns. mark any local vars + * as needing to move to the heap. + */ void -allocparams(void) +addrescapes(Node *n) { - NodeList *l; - Node *n; - uint32 w; - Sym *s; - int lno; + char buf[100]; + Node *oldfn; + + switch(n->op) { + default: + // probably a type error already. + // dump("addrescapes", n); + break; + + case ONAME: + if(n == nodfp) + break; + + // if this is a tmpname (PAUTO), it was tagged by tmpname as not escaping. + // on PPARAM it means something different. + if(n->class == PAUTO && n->esc == EscNever) + break; + + if(debug['s'] && n->esc != EscUnknown) + fatal("without escape analysis, only PAUTO's should have esc: %N", n); - if(stksize < 0) - fatal("allocparams not during code generation"); - - /* - * allocate (set xoffset) the stack - * slots for all automatics. - * allocated starting at -w down. - */ - lno = lineno; - for(l=curfn->dcl; l; l=l->next) { - n = l->n; - if(n->op == ONAME && n->class == PHEAP-1) { - // heap address variable; finish the job - // started in addrescapes. - s = n->sym; - tempname(n, n->type); - n->sym = s; + switch(n->class) { + case PPARAMREF: + addrescapes(n->defn); + break; + case PPARAM: + case PPARAMOUT: + // if func param, need separate temporary + // to hold heap pointer. + // the function type has already been checked + // (we're in the function body) + // so the param already has a valid xoffset. + + // expression to refer to stack copy + n->stackparam = nod(OPARAM, n, N); + n->stackparam->type = n->type; + n->stackparam->addable = 1; + if(n->xoffset == BADWIDTH) + fatal("addrescapes before param assignment"); + n->stackparam->xoffset = n->xoffset; + // fallthrough + + case PAUTO: + n->class |= PHEAP; + n->addable = 0; + n->ullman = 2; + n->xoffset = 0; + + // create stack variable to hold pointer to heap + oldfn = curfn; + curfn = n->curfn; + n->heapaddr = temp(ptrto(n->type)); + snprint(buf, sizeof buf, "&%S", n->sym); + n->heapaddr->sym = lookup(buf); + n->heapaddr->orig->sym = n->heapaddr->sym; + if(!debug['s']) + n->esc = EscHeap; + if(debug['m']) + print("%L: moved to heap: %hN\n", n->lineno, n); + curfn = oldfn; + break; } - if(n->op != ONAME || n->class != PAUTO) - continue; - if (n->xoffset != BADWIDTH) - continue; - if(n->type == T) - continue; - dowidth(n->type); - w = n->type->width; - if(w >= MAXWIDTH) - fatal("bad width"); - stksize += w; - stksize = rnd(stksize, n->type->align); - if(thechar == '5') - stksize = rnd(stksize, widthptr); - n->xoffset = -stksize; + break; + + case OIND: + case ODOTPTR: + break; + + case ODOT: + case OINDEX: + // ODOTPTR has already been introduced, + // so these are the non-pointer ODOT and OINDEX. + // In &x[0], if x is a slice, then x does not + // escape--the pointer inside x does, but that + // is always a heap pointer anyway. + if(!isslice(n->left->type)) + addrescapes(n->left); + break; } - lineno = lno; } void @@ -197,7 +242,7 @@ stmtlabel(Node *n) if(n->sym != S) if((lab = n->sym->label) != L) if(lab->def != N) - if(lab->def->right == n) + if(lab->def->defn == n) return lab; return L; } @@ -227,7 +272,6 @@ gen(Node *n) if(n == N) goto ret; - p3 = pc; // save pc for loop labels if(n->ninit) genlist(n->ninit); @@ -266,13 +310,13 @@ gen(Node *n) if(lab->labelpc == P) lab->labelpc = pc; - if(n->right) { - switch(n->right->op) { + if(n->defn) { + switch(n->defn->op) { case OFOR: case OSWITCH: case OSELECT: // so stmtlabel can find the label - n->right->sym = lab->sym; + n->defn->sym = lab->sym; } } break; @@ -486,7 +530,7 @@ cgen_callmeth(Node *n, int proc) /* * generate code to start new proc running call n. */ -void +static void cgen_proc(Node *n, int proc) { switch(n->left->op) { @@ -598,15 +642,12 @@ cgen_as(Node *nl, Node *nr) Type *tl; int iszer; - if(nl == N) - return; - if(debug['g']) { dump("cgen_as", nl); dump("cgen_as = ", nr); } - if(isblank(nl)) { + if(nl == N || isblank(nl)) { cgen_discard(nr); return; } @@ -748,12 +789,8 @@ tempname(Node *nn, Type *t) { Node *n; Sym *s; - uint32 w; - if(stksize < 0) - fatal("tempname not during code generation"); - - if (curfn == N) + if(curfn == N) fatal("no curfn for tempname"); if(t == T) { @@ -772,19 +809,21 @@ tempname(Node *nn, Type *t) n->class = PAUTO; n->addable = 1; n->ullman = 1; - n->noescape = 1; + n->esc = EscNever; n->curfn = curfn; curfn->dcl = list(curfn->dcl, n); dowidth(t); - w = t->width; - stksize += w; - stksize = rnd(stksize, t->align); - if(thechar == '5') - stksize = rnd(stksize, widthptr); - n->xoffset = -stksize; - - // print("\ttmpname (%d): %N\n", stksize, n); - + n->xoffset = 0; *nn = *n; } + +Node* +temp(Type *t) +{ + Node *n; + + n = nod(OXXX, N, N); + tempname(n, t); + return n; +} diff --git a/src/cmd/gc/go.h b/src/cmd/gc/go.h index da0fb5146..f72799420 100644 --- a/src/cmd/gc/go.h +++ b/src/cmd/gc/go.h @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include -#include #include #undef OAPPEND @@ -155,7 +153,6 @@ struct Type { uchar etype; uchar chan; - uchar recur; // to detect loops uchar trecur; // to detect loops uchar printed; uchar embedded; // TFIELD embedded type @@ -203,8 +200,50 @@ struct Type }; #define T ((Type*)0) +typedef struct InitEntry InitEntry; +typedef struct InitPlan InitPlan; + +struct InitEntry +{ + vlong xoffset; // struct, array only + Node *key; // map only + Node *expr; +}; + +struct InitPlan +{ + vlong lit; // bytes of initialized non-zero literals + vlong zero; // bytes of zeros + vlong expr; // bytes of run-time computed expressions + + InitEntry *e; + int len; + int cap; +}; + +enum +{ + EscUnknown, + EscHeap, + EscScope, + EscNone, + EscNever, +}; + struct Node { + // Tree structure. + // Generic recursive walks should follow these fields. + Node* left; + Node* right; + Node* ntest; + Node* nincr; + NodeList* ninit; + NodeList* nbody; + NodeList* nelse; + NodeList* list; + NodeList* rlist; + uchar op; uchar ullman; // sethi/ullman number uchar addable; // type of addressability - 0 is not addressable @@ -215,41 +254,24 @@ struct Node uchar embedded; // ODCLFIELD embedded type uchar colas; // OAS resulting from := uchar diag; // already printed error about this - uchar noescape; // ONAME never move to heap + uchar esc; // EscXXX uchar funcdepth; uchar builtin; // built-in name, like len or close uchar walkdef; uchar typecheck; uchar local; + uchar dodata; uchar initorder; - uchar dodata; // compile literal assignment as data statement uchar used; uchar isddd; - uchar pun; // don't registerize variable ONAME uchar readonly; uchar implicit; // don't show in printout // most nodes - Node* left; - Node* right; Type* type; Type* realtype; // as determined by typecheck - NodeList* list; - NodeList* rlist; Node* orig; // original form, for printing, and tracking copies of ONAMEs - // for-body - NodeList* ninit; - Node* ntest; - Node* nincr; - NodeList* nbody; - - // if-body - NodeList* nelse; - - // cases - Node* ncase; - // func Node* nname; Node* shortname; @@ -263,9 +285,10 @@ struct Node // ONAME Node* ntype; - Node* defn; + Node* defn; // ONAME: initializing assignment; OLABEL: labeled statement Node* pack; // real package for import . names Node* curfn; // function for local variables + Type* paramfld; // TFIELD for this PPARAM // ONAME func param with PHEAP Node* heapaddr; // temp holding heap address of param @@ -278,6 +301,13 @@ struct Node // OPACK Pkg* pkg; + + // OARRAYLIT, OMAPLIT, OSTRUCTLIT. + InitPlan* initplan; + + // Escape analysis. + NodeList* escflowsrc; // flow(this, src) + int escloopdepth; // -1: global, 0: not set, function top level:1, increased inside function for every loop or label to mark scopes Sym* sym; // various int32 vargen; // unique name for OTYPE/ONAME @@ -287,9 +317,25 @@ struct Node int32 stkdelta; // offset added by stack frame compaction phase. int32 ostk; int32 iota; + uint32 walkgen; }; #define N ((Node*)0) -EXTERN int32 walkgen; + +/* + * Every node has a walkgen field. + * If you want to do a traversal of a node graph that + * might contain duplicates and want to avoid + * visiting the same nodes twice, increment walkgen + * before starting. Then before processing a node, do + * + * if(n->walkgen == walkgen) + * return; + * n->walkgen = walkgen; + * + * Such a walk cannot call another such walk recursively, + * because of the use of the global walkgen. + */ +EXTERN uint32 walkgen; struct NodeList { @@ -374,7 +420,6 @@ enum OADDR, OANDAND, OAPPEND, - OARRAY, OARRAYBYTESTR, OARRAYRUNESTR, OSTRARRAYBYTE, OSTRARRAYRUNE, OAS, OAS2, OAS2MAPW, OAS2FUNC, OAS2RECV, OAS2MAPR, OAS2DOTTYPE, OASOP, @@ -444,6 +489,7 @@ enum // misc ODDD, + ODDDARG, // for back ends OCMP, ODEC, OEXTEND, OINC, OREGISTER, OINDREG, @@ -561,7 +607,6 @@ typedef struct Var Var; struct Var { vlong offset; - Sym* sym; Sym* gotype; Node* node; int width; @@ -644,6 +689,7 @@ struct Magic }; typedef struct Prog Prog; +#pragma incomplete Prog struct Label { @@ -727,6 +773,7 @@ EXTERN Pkg* phash[128]; EXTERN int tptr; // either TPTR32 or TPTR64 extern char* runtimeimport; extern char* unsafeimport; +EXTERN char* myimportpath; EXTERN Idir* idirs; EXTERN Type* types[NTYPE]; @@ -910,6 +957,11 @@ void typedcl2(Type *pt, Type *t); Node* typenod(Type *t); NodeList* variter(NodeList *vl, Node *t, NodeList *el); +/* + * esc.c + */ +void escapes(void); + /* * export.c */ @@ -927,7 +979,7 @@ Type* pkgtype(Sym *s); /* * gen.c */ -void allocparams(void); +void addrescapes(Node *n); void cgen_as(Node *nl, Node *nr); void cgen_callmeth(Node *n, int proc); void clearlabels(void); @@ -937,6 +989,7 @@ void gen(Node *n); void genlist(NodeList *l); Node* sysfunc(char *name); void tempname(Node *n, Type *t); +Node* temp(Type*); /* * init.c @@ -1050,6 +1103,7 @@ void dumptypestructs(void); Type* methodfunc(Type *f, Type*); Node* typename(Type *t); Sym* typesym(Type *t); +int haspointers(Type *t); /* * select.c @@ -1170,6 +1224,7 @@ uint32 typehash(Type *t); void ullmancalc(Node *n); void umagic(Magic *m); void warn(char *fmt, ...); +void warnl(int line, char *fmt, ...); void yyerror(char *fmt, ...); void yyerrorl(int line, char *fmt, ...); @@ -1274,6 +1329,24 @@ Prog* unpatch(Prog*); void zfile(Biobuf *b, char *p, int n); void zhist(Biobuf *b, int line, vlong offset); void zname(Biobuf *b, Sym *s, int t); -void data(void); -void text(void); +#pragma varargck type "A" int +#pragma varargck type "B" Mpint* +#pragma varargck type "D" Addr* +#pragma varargck type "lD" Addr* +#pragma varargck type "E" int +#pragma varargck type "F" Mpflt* +#pragma varargck type "J" Node* +#pragma varargck type "L" int +#pragma varargck type "L" uint +#pragma varargck type "N" Node* +#pragma varargck type "O" uint +#pragma varargck type "P" Prog* +#pragma varargck type "Q" Bits +#pragma varargck type "R" int +#pragma varargck type "S" Sym* +#pragma varargck type "lS" Sym* +#pragma varargck type "T" Type* +#pragma varargck type "lT" Type* +#pragma varargck type "Y" char* +#pragma varargck type "Z" Strlit* diff --git a/src/cmd/gc/go.y b/src/cmd/gc/go.y index 4c7fe6068..a5e92bd4d 100644 --- a/src/cmd/gc/go.y +++ b/src/cmd/gc/go.y @@ -18,7 +18,9 @@ */ %{ +#include #include /* if we don't, bison will, and go.h re-#defines getc */ +#include #include "go.h" static void fixlbrace(int); @@ -241,11 +243,11 @@ import_package: importpkg->name = $2->name; pkglookup($2->name, nil)->npkg++; } else if(strcmp(importpkg->name, $2->name) != 0) - yyerror("conflicting names %s and %s for package %Z", importpkg->name, $2->name, importpkg->path); + yyerror("conflicting names %s and %s for package \"%Z\"", importpkg->name, $2->name, importpkg->path); importpkg->direct = 1; if(safemode && !curio.importsafe) - yyerror("cannot import unsafe package %Z", importpkg->path); + yyerror("cannot import unsafe package \"%Z\"", importpkg->path); } import_safety: @@ -1494,7 +1496,7 @@ non_dcl_stmt: { NodeList *l; - $1->right = $4; + $1->defn = $4; l = list1($1); if($4) l = list(l, $4); @@ -1684,7 +1686,11 @@ hidden_import: p->name = $2->name; pkglookup($2->name, nil)->npkg++; } else if(strcmp(p->name, $2->name) != 0) - yyerror("conflicting names %s and %s for package %Z", p->name, $2->name, p->path); + yyerror("conflicting names %s and %s for package \"%Z\"", p->name, $2->name, p->path); + if(!incannedimport && myimportpath != nil && strcmp($3.u.sval->s, myimportpath) == 0) { + yyerror("import \"%Z\": package depends on \"%Z\" (import cycle)", importpkg->path, $3.u.sval); + errorexit(); + } } | LVAR hidden_pkg_importsym hidden_type ';' { diff --git a/src/cmd/gc/init.c b/src/cmd/gc/init.c index 8818db08c..da69e41ae 100644 --- a/src/cmd/gc/init.c +++ b/src/cmd/gc/init.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" /* diff --git a/src/cmd/gc/lex.c b/src/cmd/gc/lex.c index 29b6d27ff..0290fb131 100644 --- a/src/cmd/gc/lex.c +++ b/src/cmd/gc/lex.c @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#define EXTERN +#include +#include #include "go.h" #include "y.tab.h" #include @@ -64,7 +65,7 @@ yy_isalnum(int c) #define isalpha use_yy_isalpha_instead_of_isalpha #define isalnum use_yy_isalnum_instead_of_isalnum -#define DBG if(!debug['x']);else print +#define DBG if(!debug['x']){}else print enum { EOF = -1, @@ -77,22 +78,29 @@ usage(void) print("flags:\n"); // -A is allow use of "any" type, for bootstrapping print(" -I DIR search for packages in DIR\n"); + print(" -L show full path in file:line prints\n"); + print(" -N disable optimizer\n"); + print(" -S print the assembly language\n"); + print(" -V print the compiler version\n"); print(" -d print declarations\n"); print(" -e no limit on number of errors printed\n"); print(" -f print stack frame structure\n"); print(" -h panic on an error\n"); + print(" -m print about moves to heap\n"); print(" -o file specify output file\n"); - print(" -S print the assembly language\n"); - print(" -V print the compiler version\n"); + print(" -p assumed import path for this code\n"); + print(" -s disable escape analysis\n"); print(" -u disable package unsafe\n"); print(" -w print the parse tree after typing\n"); print(" -x print lex tokens\n"); - exit(0); + exits(0); } void fault(int s) { + USED(s); + // If we've already complained about things // in the program, don't bother complaining // about the seg fault too; let the user clean up @@ -108,9 +116,11 @@ main(int argc, char *argv[]) int i, c; NodeList *l; char *p; - + +#ifdef SIGBUS signal(SIGBUS, fault); signal(SIGSEGV, fault); +#endif localpkg = mkpkg(strlit("")); localpkg->prefix = "\"\""; @@ -145,6 +155,10 @@ main(int argc, char *argv[]) case 'o': outfile = EARGF(usage()); break; + + case 'p': + myimportpath = EARGF(usage()); + break; case 'I': addidir(EARGF(usage())); @@ -156,7 +170,7 @@ main(int argc, char *argv[]) case 'V': print("%cg version %s\n", thechar, getgoversion()); - exit(0); + exits(0); } ARGEND if(argc < 1) @@ -236,24 +250,24 @@ main(int argc, char *argv[]) if(debug['f']) frame(1); - // Process top-level declarations in four phases. + // Process top-level declarations in phases. // Phase 1: const, type, and names and types of funcs. // This will gather all the information about types // and methods but doesn't depend on any of it. - // Phase 2: Variable assignments. - // To check interface assignments, depends on phase 1. - // Phase 3: Type check function bodies. - // Phase 4: Compile function bodies. defercheckwidth(); for(l=xtop; l; l=l->next) if(l->n->op != ODCL && l->n->op != OAS) typecheck(&l->n, Etop); + + // Phase 2: Variable assignments. + // To check interface assignments, depends on phase 1. for(l=xtop; l; l=l->next) if(l->n->op == ODCL || l->n->op == OAS) typecheck(&l->n, Etop); resumetypecopy(); resumecheckwidth(); + // Phase 3: Type check function bodies. for(l=xtop; l; l=l->next) { if(l->n->op == ODCLFUNC || l->n->op == OCLOSURE) { curfn = l->n; @@ -269,6 +283,11 @@ main(int argc, char *argv[]) if(nsavederrors+nerrors) errorexit(); + // Phase 3b: escape analysis. + if(!debug['s']) + escapes(); + + // Phase 4: Compile function bodies. for(l=xtop; l; l=l->next) if(l->n->op == ODCLFUNC) funccompile(l->n, 0); @@ -276,6 +295,7 @@ main(int argc, char *argv[]) if(nsavederrors+nerrors == 0) fninit(xtop); + // Phase 4b: Compile all closures. while(closures) { l = closures; closures = nil; @@ -284,6 +304,7 @@ main(int argc, char *argv[]) } } + // Phase 5: check external declarations. for(l=externdcl; l; l=l->next) if(l->n->op == ONAME) typecheck(&l->n, Erv); @@ -297,7 +318,7 @@ main(int argc, char *argv[]) errorexit(); flusherrors(); - exit(0); + exits(0); return 0; } @@ -400,8 +421,8 @@ findpkg(Strlit *name) } // local imports should be canonicalized already. - // don't want to see "container/../container/vector" - // as different from "container/vector". + // don't want to see "encoding/../encoding/base64" + // as different from "encoding/base64". q = mal(name->len+1); memmove(q, name->s, name->len); q[name->len] = '\0'; @@ -440,6 +461,8 @@ importfile(Val *f, int line) Strlit *path; char *cleanbuf; + USED(line); + // TODO(rsc): don't bother reloading imports more than once? if(f->ctype != CTSTR) { @@ -461,6 +484,11 @@ importfile(Val *f, int line) errorexit(); } + if(myimportpath != nil && strcmp(f->u.sval->s, myimportpath) == 0) { + yyerror("import \"%Z\" while compiling that package (import cycle)", f->u.sval); + errorexit(); + } + if(strcmp(f->u.sval->s, "unsafe") == 0) { if(safemode) { yyerror("cannot import package unsafe"); @@ -482,14 +510,14 @@ importfile(Val *f, int line) } if(!findpkg(path)) { - yyerror("can't find import: %Z", f->u.sval); + yyerror("can't find import: \"%Z\"", f->u.sval); errorexit(); } importpkg = mkpkg(path); imp = Bopen(namebuf, OREAD); if(imp == nil) { - yyerror("can't open import: %Z: %r", f->u.sval); + yyerror("can't open import: \"%Z\": %r", f->u.sval); errorexit(); } file = strdup(namebuf); @@ -546,7 +574,7 @@ importfile(Val *f, int line) continue; return; } - yyerror("no import in: %Z", f->u.sval); + yyerror("no import in \"%Z\"", f->u.sval); unimportfile(); } @@ -665,7 +693,6 @@ l0: ep = lexbuf+sizeof lexbuf; *cp++ = c; c = c1; - c1 = 0; goto casedot; } if(c1 == '.') { @@ -1056,7 +1083,6 @@ talph: return s->lexical; tnum: - c1 = 0; cp = lexbuf; ep = lexbuf+sizeof lexbuf; if(c != '0') { @@ -1740,7 +1766,6 @@ lexfini(void) } nodfp = nod(ONAME, N, N); - nodfp->noescape = 1; nodfp->type = types[TINT32]; nodfp->xoffset = 0; nodfp->class = PPARAM; @@ -1923,7 +1948,7 @@ mkpackage(char* pkgname) // errors if a conflicting top-level name is // introduced by a different file. if(!s->def->used && !nsyntaxerrors) - yyerrorl(s->def->lineno, "imported and not used: %Z", s->def->pkg->path); + yyerrorl(s->def->lineno, "imported and not used: \"%Z\"", s->def->pkg->path); s->def = N; continue; } @@ -1931,7 +1956,7 @@ mkpackage(char* pkgname) // throw away top-level name left over // from previous import . "x" if(s->def->pack != N && !s->def->pack->used && !nsyntaxerrors) { - yyerrorl(s->def->pack->lineno, "imported and not used: %Z", s->def->pack->pkg->path); + yyerrorl(s->def->pack->lineno, "imported and not used: \"%Z\"", s->def->pack->pkg->path); s->def->pack->used = 1; } s->def = N; diff --git a/src/cmd/gc/md5.c b/src/cmd/gc/md5.c index 7cea1a6cf..5856aab51 100644 --- a/src/cmd/gc/md5.c +++ b/src/cmd/gc/md5.c @@ -5,6 +5,8 @@ // 64-bit MD5 (does full MD5 but returns 64 bits only). // Translation of ../../pkg/crypto/md5/md5*.go. +#include +#include #include "go.h" #include "md5.h" diff --git a/src/cmd/gc/mparith1.c b/src/cmd/gc/mparith1.c index 6cd4e2500..2b7307e1a 100644 --- a/src/cmd/gc/mparith1.c +++ b/src/cmd/gc/mparith1.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" /// uses arithmetic diff --git a/src/cmd/gc/mparith2.c b/src/cmd/gc/mparith2.c index 403255005..71cc29c99 100644 --- a/src/cmd/gc/mparith2.c +++ b/src/cmd/gc/mparith2.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" // @@ -349,6 +351,7 @@ mporfixfix(Mpint *a, Mpint *b) int i; long x, *a1, *b1; + x = 0; if(a->ovf || b->ovf) { yyerror("ovf in mporfixfix"); mpmovecfix(a, 0); @@ -383,6 +386,7 @@ mpandfixfix(Mpint *a, Mpint *b) int i; long x, *a1, *b1; + x = 0; if(a->ovf || b->ovf) { yyerror("ovf in mpandfixfix"); mpmovecfix(a, 0); @@ -417,6 +421,7 @@ mpandnotfixfix(Mpint *a, Mpint *b) int i; long x, *a1, *b1; + x = 0; if(a->ovf || b->ovf) { yyerror("ovf in mpandnotfixfix"); mpmovecfix(a, 0); @@ -451,6 +456,7 @@ mpxorfixfix(Mpint *a, Mpint *b) int i; long x, *a1, *b1; + x = 0; if(a->ovf || b->ovf) { yyerror("ovf in mporfixfix"); mpmovecfix(a, 0); diff --git a/src/cmd/gc/mparith3.c b/src/cmd/gc/mparith3.c index b11a4f5f1..0c6c5a03b 100644 --- a/src/cmd/gc/mparith3.c +++ b/src/cmd/gc/mparith3.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" /* diff --git a/src/cmd/gc/obj.c b/src/cmd/gc/obj.c index f34fc76c8..d6fe6f65d 100644 --- a/src/cmd/gc/obj.c +++ b/src/cmd/gc/obj.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" /* @@ -29,10 +31,6 @@ dumpobj(void) outhist(bout); - // add nil plist w AEND to catch - // auto-generated trampolines, data - newplist(); - dumpglobls(); dumptypestructs(); dumpdata(); @@ -127,6 +125,7 @@ static void outhist(Biobuf *b) { Hist *h; + int i, depth = 0; char *p, ds[] = {'c', ':', '/', 0}; for(h = hist; h != H; h = h->link) { @@ -156,13 +155,21 @@ outhist(Biobuf *b) outzfile(b, p+1); } else { // relative name, like dir/file.go - if(h->offset == 0 && pathname && pathname[0] == '/') { + if(h->offset >= 0 && pathname && pathname[0] == '/') { zfile(b, "/", 1); // leading "/" outzfile(b, pathname+1); } outzfile(b, p); } } + if(h->offset > 0) { + //line directive + depth++; + } + } else if(depth > 0) { + for(i = 0; i < depth; i++) + zhist(b, h->line, h->offset); + depth = 0; } zhist(b, h->line, h->offset); } @@ -259,7 +266,7 @@ stringsym(char *s, int len) tmp.lit.len = len; memmove(tmp.lit.s, s, len); tmp.lit.s[len] = '\0'; - snprint(namebuf, sizeof(namebuf), "\"%Z\"", &tmp); + snprint(namebuf, sizeof(namebuf), "\"%Z\"", &tmp.lit); pkg = gostringpkg; } sym = pkglookup(namebuf, pkg); @@ -268,8 +275,7 @@ stringsym(char *s, int len) if(sym->flags & SymUniq) return sym; sym->flags |= SymUniq; - - data(); + off = 0; // string header @@ -286,7 +292,6 @@ stringsym(char *s, int len) off = duint8(sym, off, 0); // terminating NUL for runtime off = (off+widthptr-1)&~(widthptr-1); // round to pointer alignment ggloblsym(sym, off, 1); - text(); - + return sym; } diff --git a/src/cmd/gc/pgen.c b/src/cmd/gc/pgen.c index abe8ea892..d16481b66 100644 --- a/src/cmd/gc/pgen.c +++ b/src/cmd/gc/pgen.c @@ -2,10 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "gg.h" #include "opt.h" -static void compactframe(Prog* p); +static void allocauto(Prog* p); void compile(Node *fn) @@ -58,8 +60,6 @@ compile(Node *fn) if(nerrors != 0) goto ret; - allocparams(); - continpc = P; breakpc = P; @@ -113,9 +113,9 @@ compile(Node *fn) } oldstksize = stksize; - compactframe(ptxt); + allocauto(ptxt); if(0) - print("compactframe: %ld to %ld\n", oldstksize, stksize); + print("allocauto: %lld to %lld\n", oldstksize, (vlong)stksize); defframe(ptxt); @@ -145,13 +145,13 @@ cmpstackvar(Node *a, Node *b) // TODO(lvd) find out where the PAUTO/OLITERAL nodes come from. static void -compactframe(Prog* ptxt) +allocauto(Prog* ptxt) { NodeList *ll; Node* n; vlong w; - if (stksize == 0) + if(curfn->dcl == nil) return; // Mark the PAUTO's unused. @@ -188,6 +188,7 @@ compactframe(Prog* ptxt) if (n->class != PAUTO || n->op != ONAME) continue; + dowidth(n->type); w = n->type->width; if(w >= MAXWIDTH || w < 0) fatal("bad width"); diff --git a/src/cmd/gc/print.c b/src/cmd/gc/print.c index 5913e848a..37e3e7ac0 100644 --- a/src/cmd/gc/print.c +++ b/src/cmd/gc/print.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" enum @@ -139,6 +141,10 @@ exprfmt(Fmt *f, Node *n, int prec) fmtprint(f, "(%#N)", n->left); break; + case ODDDARG: + fmtprint(f, "... argument"); + break; + case OREGISTER: fmtprint(f, "%R", n->val.u.reg); break; diff --git a/src/cmd/gc/range.c b/src/cmd/gc/range.c index 5ce693ae3..5cbafd895 100644 --- a/src/cmd/gc/range.c +++ b/src/cmd/gc/range.c @@ -6,6 +6,8 @@ * range */ +#include +#include #include "go.h" void @@ -111,8 +113,6 @@ walkrange(Node *n) } v1 = n->list->n; - hv1 = N; - v2 = N; if(n->list->next) v2 = n->list->next->n; @@ -123,8 +123,7 @@ walkrange(Node *n) // no need to make a potentially expensive copy. ha = a; } else { - ha = nod(OXXX, N, N); - tempname(ha, a->type); + ha = temp(a->type); init = list(init, nod(OAS, ha, a)); } @@ -133,17 +132,14 @@ walkrange(Node *n) fatal("walkrange"); case TARRAY: - hv1 = nod(OXXX, N, n); - tempname(hv1, types[TINT]); - hn = nod(OXXX, N, N); - tempname(hn, types[TINT]); + hv1 = temp(types[TINT]); + hn = temp(types[TINT]); hp = nil; init = list(init, nod(OAS, hv1, N)); init = list(init, nod(OAS, hn, nod(OLEN, ha, N))); if(v2) { - hp = nod(OXXX, N, N); - tempname(hp, ptrto(n->type->type)); + hp = temp(ptrto(n->type->type)); tmp = nod(OINDEX, ha, nodintconst(0)); tmp->etype = 1; // no bounds check init = list(init, nod(OAS, hp, nod(OADDR, tmp, N))); @@ -168,8 +164,7 @@ walkrange(Node *n) th = typ(TARRAY); th->type = ptrto(types[TUINT8]); th->bound = (sizeof(struct Hiter) + widthptr - 1) / widthptr; - hit = nod(OXXX, N, N); - tempname(hit, th); + hit = temp(th); fn = syslook("mapiterinit", 1); argtype(fn, t->down); @@ -200,10 +195,8 @@ walkrange(Node *n) break; case TCHAN: - hv1 = nod(OXXX, N, n); - tempname(hv1, t->type); - hb = nod(OXXX, N, N); - tempname(hb, types[TBOOL]); + hv1 = temp(t->type); + hb = temp(types[TBOOL]); n->ntest = nod(ONE, hb, nodbool(0)); a = nod(OAS2RECV, N, N); @@ -215,18 +208,15 @@ walkrange(Node *n) break; case TSTRING: - ohv1 = nod(OXXX, N, N); - tempname(ohv1, types[TINT]); + ohv1 = temp(types[TINT]); - hv1 = nod(OXXX, N, N); - tempname(hv1, types[TINT]); + hv1 = temp(types[TINT]); init = list(init, nod(OAS, hv1, N)); if(v2 == N) a = nod(OAS, hv1, mkcall("stringiter", types[TINT], nil, ha, hv1)); else { - hv2 = nod(OXXX, N, N); - tempname(hv2, types[TINT]); + hv2 = temp(types[TINT]); a = nod(OAS2, N, N); a->list = list(list1(hv1), hv2); fn = syslook("stringiter2", 0); diff --git a/src/cmd/gc/reflect.c b/src/cmd/gc/reflect.c index 810787d30..ca7d08e51 100644 --- a/src/cmd/gc/reflect.c +++ b/src/cmd/gc/reflect.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" /* @@ -140,7 +142,6 @@ methods(Type *t) Type *f, *mt, *it, *this; Sig *a, *b; Sym *method; - Prog *oldlist; // named method type mt = methtype(t); @@ -156,7 +157,6 @@ methods(Type *t) // make list of methods for t, // generating code if necessary. a = nil; - oldlist = nil; for(f=mt->xmethod; f; f=f->down) { if(f->type->etype != TFUNC) continue; @@ -195,8 +195,6 @@ methods(Type *t) if(!(a->isym->flags & SymSiggen)) { a->isym->flags |= SymSiggen; if(!eqtype(this, it) || this->width < types[tptr]->width) { - if(oldlist == nil) - oldlist = pc; // Is okay to call genwrapper here always, // but we can generate more efficient code // using genembedtramp if all that is necessary @@ -212,8 +210,6 @@ methods(Type *t) if(!(a->tsym->flags & SymSiggen)) { a->tsym->flags |= SymSiggen; if(!eqtype(this, t)) { - if(oldlist == nil) - oldlist = pc; if(isptr[t->etype] && isptr[this->etype] && f->embedded && !isifacemethod(f->type)) genembedtramp(t, f, a->tsym, 0); @@ -223,16 +219,6 @@ methods(Type *t) } } - // restore data output - if(oldlist) { - // old list ended with AEND; change to ANOP - // so that the trampolines that follow can be found. - nopout(oldlist); - - // start new data list - newplist(); - } - return lsort(a, sigcmp); } @@ -245,11 +231,9 @@ imethods(Type *t) Sig *a, *all, *last; Type *f; Sym *method, *isym; - Prog *oldlist; all = nil; last = nil; - oldlist = nil; for(f=t->type; f; f=f->down) { if(f->etype != TFIELD) fatal("imethods: not field"); @@ -287,21 +271,9 @@ imethods(Type *t) isym = methodsym(method, t, 0); if(!(isym->flags & SymSiggen)) { isym->flags |= SymSiggen; - if(oldlist == nil) - oldlist = pc; genwrapper(t, f, isym, 0); } } - - if(oldlist) { - // old list ended with AEND; change to ANOP - // so that the trampolines that follow can be found. - nopout(oldlist); - - // start new data list - newplist(); - } - return all; } @@ -528,7 +500,7 @@ typestruct(Type *t) return pkglookup(name, typepkg); } -static int +int haspointers(Type *t) { Type *t1; @@ -584,7 +556,6 @@ dcommontype(Sym *s, int ot, Type *t) dowidth(t); - sptr = nil; if(t->sym != nil && !isptr[t->etype]) sptr = dtypesym(ptrto(t)); else diff --git a/src/cmd/gc/select.c b/src/cmd/gc/select.c index 909ad3aa4..8ace1d4ee 100644 --- a/src/cmd/gc/select.c +++ b/src/cmd/gc/select.c @@ -6,6 +6,8 @@ * select */ +#include +#include #include "go.h" void @@ -59,7 +61,7 @@ typecheckselect(Node *sel) break; case OAS2RECV: - // convert x, ok = <-c into OSELRECV(x, <-c) with ntest=ok + // convert x, ok = <-c into OSELRECV2(x, <-c) with ntest=ok if(n->right->op != ORECV) { yyerror("select assignment must have receive on right hand side"); break; @@ -73,6 +75,7 @@ typecheckselect(Node *sel) case ORECV: // convert <-c into OSELRECV(N, <-c) n = nod(OSELRECV, N, n); + n->typecheck = 1; ncase->left = n; break; @@ -191,8 +194,7 @@ walkselect(Node *sel) n->ntest->etype = 1; // pointer does not escape typecheck(&n->ntest, Erv); } else { - tmp = nod(OXXX, N, N); - tempname(tmp, types[TBOOL]); + tmp = temp(types[TBOOL]); a = nod(OADDR, tmp, N); a->etype = 1; // pointer does not escape typecheck(&a, Erv); @@ -212,8 +214,7 @@ walkselect(Node *sel) n->left->etype = 1; // pointer does not escape typecheck(&n->left, Erv); } else { - tmp = nod(OXXX, N, N); - tempname(tmp, ch->type->type); + tmp = temp(ch->type->type); a = nod(OADDR, tmp, N); a->etype = 1; // pointer does not escape typecheck(&a, Erv); @@ -284,8 +285,7 @@ walkselect(Node *sel) // generate sel-struct setlineno(sel); - var = nod(OXXX, N, N); - tempname(var, ptrto(types[TUINT8])); + var = temp(ptrto(types[TUINT8])); r = nod(OAS, var, mkcall("newselect", var->type, nil, nodintconst(sel->xoffset))); typecheck(&r, Etop); init = list(init, r); diff --git a/src/cmd/gc/sinit.c b/src/cmd/gc/sinit.c index 917e2ae6d..4550577a4 100644 --- a/src/cmd/gc/sinit.c +++ b/src/cmd/gc/sinit.c @@ -6,11 +6,24 @@ * static initialization */ +#include +#include #include "go.h" +enum +{ + InitNotStarted = 0, + InitDone = 1, + InitPending = 2, +}; + +static int iszero(Node*); +static void initplan(Node*); static NodeList *initlist; static void init2(Node*, NodeList**); static void init2list(NodeList*, NodeList**); +static int staticinit(Node*, NodeList**); +static Node *staticname(Type*, int); static void init1(Node *n, NodeList **out) @@ -31,16 +44,16 @@ init1(Node *n, NodeList **out) case PFUNC: break; default: - if(isblank(n) && n->defn != N && !n->defn->initorder) { - n->defn->initorder = 1; + if(isblank(n) && n->defn != N && n->defn->initorder == InitNotStarted) { + n->defn->initorder = InitDone; *out = list(*out, n->defn); } return; } - if(n->initorder == 1) + if(n->initorder == InitDone) return; - if(n->initorder == 2) { + if(n->initorder == InitPending) { if(n->class == PFUNC) return; @@ -63,7 +76,7 @@ init1(Node *n, NodeList **out) print("\t%L %S\n", n->lineno, n->sym); errorexit(); } - n->initorder = 2; + n->initorder = InitPending; l = malloc(sizeof *l); l->next = initlist; l->n = n; @@ -84,20 +97,38 @@ init1(Node *n, NodeList **out) case OAS: if(n->defn->left != n) goto bad; + /* n->defn->dodata = 1; init1(n->defn->right, out); if(debug['j']) print("%S\n", n->sym); *out = list(*out, n->defn); break; + */ + if(1) { + init1(n->defn->right, out); + if(debug['j']) + print("%S\n", n->sym); + if(!staticinit(n, out)) { +if(debug['%']) dump("nonstatic", n->defn); + *out = list(*out, n->defn); + } + } else if(0) { + n->defn->dodata = 1; + init1(n->defn->right, out); + if(debug['j']) + print("%S\n", n->sym); + *out = list(*out, n->defn); + } + break; case OAS2FUNC: case OAS2MAPR: case OAS2DOTTYPE: case OAS2RECV: - if(n->defn->initorder) + if(n->defn->initorder != InitNotStarted) break; - n->defn->initorder = 1; + n->defn->initorder = InitDone; for(l=n->defn->rlist; l; l=l->next) init1(l->n, out); *out = list(*out, n->defn); @@ -109,7 +140,7 @@ init1(Node *n, NodeList **out) if(l->n != n) fatal("bad initlist"); free(l); - n->initorder = 1; + n->initorder = InitDone; return; bad: @@ -121,7 +152,7 @@ bad: static void init2(Node *n, NodeList **out) { - if(n == N || n->initorder == 1) + if(n == N || n->initorder == InitDone) return; init1(n, out); init2(n->left, out); @@ -141,7 +172,6 @@ init2list(NodeList *l, NodeList **out) init2(l->n, out); } - static void initreorder(NodeList *l, NodeList **out) { @@ -165,12 +195,220 @@ NodeList* initfix(NodeList *l) { NodeList *lout; + int lno; lout = nil; + lno = lineno; initreorder(l, &lout); + lineno = lno; return lout; } +/* + * compilation of top-level (static) assignments + * into DATA statements if at all possible. + */ + +static int staticassign(Node*, Node*, NodeList**); + +static int +staticinit(Node *n, NodeList **out) +{ + Node *l, *r; + + if(n->op != ONAME || n->class != PEXTERN || n->defn == N || n->defn->op != OAS) + fatal("staticinit"); + + lineno = n->lineno; + l = n->defn->left; + r = n->defn->right; + return staticassign(l, r, out); +} + +// like staticassign but we are copying an already +// initialized value r. +static int +staticcopy(Node *l, Node *r, NodeList **out) +{ + int i; + InitEntry *e; + InitPlan *p; + Node *a, *ll, *rr, *orig, n1; + + if(r->op != ONAME || r->class != PEXTERN || r->sym->pkg != localpkg) + return 0; + if(r->defn == N) // zeroed + return 1; + if(r->defn->op != OAS) + return 0; + orig = r; + r = r->defn->right; + + switch(r->op) { + case ONAME: + if(staticcopy(l, r, out)) + return 1; + *out = list(*out, nod(OAS, l, r)); + return 1; + + case OLITERAL: + if(iszero(r)) + return 1; + gdata(l, r, l->type->width); + return 1; + + case OADDR: + switch(r->left->op) { + case ONAME: + gdata(l, r, l->type->width); + return 1; + case OARRAYLIT: + case OSTRUCTLIT: + case OMAPLIT: + // copy pointer + gdata(l, nod(OADDR, r->nname, N), l->type->width); + return 1; + } + break; + + case OARRAYLIT: + if(isslice(r->type)) { + // copy slice + a = r->nname; + n1 = *l; + n1.xoffset = l->xoffset + Array_array; + gdata(&n1, nod(OADDR, a, N), widthptr); + n1.xoffset = l->xoffset + Array_nel; + gdata(&n1, r->right, 4); + n1.xoffset = l->xoffset + Array_cap; + gdata(&n1, r->right, 4); + return 1; + } + // fall through + case OSTRUCTLIT: + p = r->initplan; + n1 = *l; + for(i=0; ilen; i++) { + e = &p->e[i]; + n1.xoffset = l->xoffset + e->xoffset; + n1.type = e->expr->type; + if(e->expr->op == OLITERAL) + gdata(&n1, e->expr, n1.type->width); + else if(staticassign(&n1, e->expr, out)) { + // Done + } else { + // Requires computation, but we're + // copying someone else's computation. + ll = nod(OXXX, N, N); + *ll = n1; + rr = nod(OXXX, N, N); + *rr = *orig; + rr->type = ll->type; + rr->xoffset += e->xoffset; + *out = list(*out, nod(OAS, ll, rr)); + } + } + return 1; + } + return 0; +} + +static int +staticassign(Node *l, Node *r, NodeList **out) +{ + Node *a, n1; + Type *ta; + InitPlan *p; + InitEntry *e; + int i; + + switch(r->op) { + default: + //dump("not static", r); + break; + + case ONAME: + if(r->class == PEXTERN && r->sym->pkg == localpkg) + return staticcopy(l, r, out); + break; + + case OLITERAL: + if(iszero(r)) + return 1; + gdata(l, r, l->type->width); + return 1; + + case OADDR: + switch(r->left->op) { + default: + //dump("not static addr", r); + break; + + case ONAME: + gdata(l, r, l->type->width); + return 1; + + case OARRAYLIT: + case OMAPLIT: + case OSTRUCTLIT: + // Init pointer. + a = staticname(r->left->type, 1); + r->nname = a; + gdata(l, nod(OADDR, a, N), l->type->width); + // Init underlying literal. + if(!staticassign(a, r->left, out)) + *out = list(*out, nod(OAS, a, r->left)); + return 1; + } + break; + + case OARRAYLIT: + initplan(r); + if(isslice(r->type)) { + // Init slice. + ta = typ(TARRAY); + ta->type = r->type->type; + ta->bound = mpgetfix(r->right->val.u.xval); + a = staticname(ta, 1); + r->nname = a; + n1 = *l; + n1.xoffset = l->xoffset + Array_array; + gdata(&n1, nod(OADDR, a, N), widthptr); + n1.xoffset = l->xoffset + Array_nel; + gdata(&n1, r->right, 4); + n1.xoffset = l->xoffset + Array_cap; + gdata(&n1, r->right, 4); + // Fall through to init underlying array. + l = a; + } + // fall through + case OSTRUCTLIT: + initplan(r); + p = r->initplan; + n1 = *l; + for(i=0; ilen; i++) { + e = &p->e[i]; + n1.xoffset = l->xoffset + e->xoffset; + n1.type = e->expr->type; + if(e->expr->op == OLITERAL) + gdata(&n1, e->expr, n1.type->width); + else if(staticassign(&n1, e->expr, out)) { + // done + } else { + a = nod(OXXX, N, N); + *a = n1; + *out = list(*out, nod(OAS, a, e->expr)); + } + } + return 1; + + case OMAPLIT: + // TODO: Table-driven map insert. + break; + } + return 0; +} + /* * from here down is the walk analysis * of composite literals. @@ -408,7 +646,6 @@ slicelit(int ctxt, Node *n, Node *var, NodeList **init) dowidth(t); if(ctxt != 0) { - // put everything into static array vstat = staticname(t, ctxt); arraylit(ctxt, 1, n, vstat, init); @@ -452,12 +689,17 @@ slicelit(int ctxt, Node *n, Node *var, NodeList **init) } // make new auto *array (3 declare) - vauto = nod(OXXX, N, N); - tempname(vauto, ptrto(t)); - - // set auto to point at new heap (3 assign) - a = nod(ONEW, N, N); - a->list = list1(typenod(t)); + vauto = temp(ptrto(t)); + + // set auto to point at new temp or heap (3 assign) + if(n->esc == EscNone) { + a = temp(t); + *init = list(*init, nod(OAS, a, N)); // zero new temp + a = nod(OADDR, a, N); + } else { + a = nod(ONEW, N, N); + a->list = list1(typenod(t)); + } a = nod(OAS, vauto, a); typecheck(&a, Etop); walkexpr(&a, init); @@ -522,6 +764,7 @@ maplit(int ctxt, Node *n, Node *var, NodeList **init) Node *vstat, *index, *value; Sym *syma, *symb; +USED(ctxt); ctxt = 0; // make the map var @@ -545,7 +788,6 @@ ctxt = 0; b++; } - t = T; if(b != 0) { // build type [count]struct { a Tindex, b Tvalue } t = n->type; @@ -615,8 +857,7 @@ ctxt = 0; // for i = 0; i < len(vstat); i++ { // map[vstat[i].a] = vstat[i].b // } - index = nod(OXXX, N, N); - tempname(index, types[TINT]); + index = temp(types[TINT]); a = nod(OINDEX, vstat, index); a->etype = 1; // no bounds checking @@ -917,18 +1158,15 @@ gen_as_init(Node *n) case TPTR64: case TFLOAT32: case TFLOAT64: - gused(N); // in case the data is the dest of a goto gdata(&nam, nr, nr->type->width); break; case TCOMPLEX64: case TCOMPLEX128: - gused(N); // in case the data is the dest of a goto gdatacomplex(&nam, nr->val.u.cval); break; case TSTRING: - gused(N); // in case the data is the dest of a goto gdatastring(&nam, nr->val.u.sval); break; } @@ -969,3 +1207,151 @@ no: return 0; } +static int iszero(Node*); +static int isvaluelit(Node*); +static InitEntry* entry(InitPlan*); +static void addvalue(InitPlan*, vlong, Node*, Node*); + +static void +initplan(Node *n) +{ + InitPlan *p; + Node *a; + NodeList *l; + + if(n->initplan != nil) + return; + p = mal(sizeof *p); + n->initplan = p; + switch(n->op) { + default: + fatal("initplan"); + case OARRAYLIT: + for(l=n->list; l; l=l->next) { + a = l->n; + if(a->op != OKEY || !smallintconst(a->left)) + fatal("initplan arraylit"); + addvalue(p, n->type->type->width*mpgetfix(a->left->val.u.xval), N, a->right); + } + break; + case OSTRUCTLIT: + for(l=n->list; l; l=l->next) { + a = l->n; + if(a->op != OKEY || a->left->type == T) + fatal("initplan structlit"); + addvalue(p, a->left->type->width, N, a->right); + } + break; + case OMAPLIT: + for(l=n->list; l; l=l->next) { + a = l->n; + if(a->op != OKEY) + fatal("initplan maplit"); + addvalue(p, -1, a->left, a->right); + } + break; + } +} + +static void +addvalue(InitPlan *p, vlong xoffset, Node *key, Node *n) +{ + int i; + InitPlan *q; + InitEntry *e; + + USED(key); + + // special case: zero can be dropped entirely + if(iszero(n)) { + p->zero += n->type->width; + return; + } + + // special case: inline struct and array (not slice) literals + if(isvaluelit(n)) { + initplan(n); + q = n->initplan; + for(i=0; ilen; i++) { + e = entry(p); + *e = q->e[i]; + e->xoffset += xoffset; + } + return; + } + + // add to plan + if(n->op == OLITERAL) + p->lit += n->type->width; + else + p->expr += n->type->width; + + e = entry(p); + e->xoffset = xoffset; + e->expr = n; +} + +static int +iszero(Node *n) +{ + NodeList *l; + + switch(n->op) { + case OLITERAL: + switch(n->val.ctype) { + default: + dump("unexpected literal", n); + fatal("iszero"); + + case CTNIL: + return 1; + + case CTSTR: + return n->val.u.sval == nil || n->val.u.sval->len == 0; + + case CTBOOL: + return n->val.u.bval == 0; + + case CTINT: + return mpcmpfixc(n->val.u.xval, 0) == 0; + + case CTFLT: + return mpcmpfltc(n->val.u.fval, 0) == 0; + + case CTCPLX: + return mpcmpfltc(&n->val.u.cval->real, 0) == 0 && mpcmpfltc(&n->val.u.cval->imag, 0) == 0; + } + break; + case OARRAYLIT: + if(isslice(n->type)) + break; + // fall through + case OSTRUCTLIT: + for(l=n->list; l; l=l->next) + if(!iszero(l->n->right)) + return 0; + return 1; + } + return 0; +} + +static int +isvaluelit(Node *n) +{ + return (n->op == OARRAYLIT && isfixedarray(n->type)) || n->op == OSTRUCTLIT; +} + +static InitEntry* +entry(InitPlan *p) +{ + if(p->len >= p->cap) { + if(p->cap == 0) + p->cap = 4; + else + p->cap *= 2; + p->e = realloc(p->e, p->cap*sizeof p->e[0]); + if(p->e == nil) + fatal("out of memory"); + } + return &p->e[p->len++]; +} diff --git a/src/cmd/gc/subr.c b/src/cmd/gc/subr.c index 1a05d43d0..9448c3ffe 100644 --- a/src/cmd/gc/subr.c +++ b/src/cmd/gc/subr.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" #include "md5.h" #include "y.tab.h" @@ -27,7 +29,7 @@ errorexit(void) flusherrors(); if(outfile) remove(outfile); - exit(1); + exits("error"); } extern int yychar; @@ -106,7 +108,7 @@ hcrash(void) if(debug['h']) { flusherrors(); if(outfile) - unlink(outfile); + remove(outfile); *(volatile int*)0 = 0; } } @@ -209,6 +211,16 @@ warn(char *fmt, ...) hcrash(); } +void +warnl(int line, char *fmt, ...) +{ + va_list arg; + + va_start(arg, fmt); + adderr(line, fmt, arg); + va_end(arg); +} + void fatal(char *fmt, ...) { @@ -393,7 +405,7 @@ importdot(Pkg *opkg, Node *pack) } if(n == 0) { // can't possibly be used - there were no symbols - yyerrorl(pack->lineno, "imported and not used: %Z", opkg->path); + yyerrorl(pack->lineno, "imported and not used: \"%Z\"", opkg->path); } } @@ -483,6 +495,7 @@ nod(int op, Node *nleft, Node *nright) n->lineno = parserline(); n->xoffset = BADWIDTH; n->orig = n; + n->curfn = curfn; return n; } @@ -1094,8 +1107,8 @@ Jconv(Fmt *fp) if(n->class != 0) { s = ""; - if (n->class & PHEAP) s = ",heap"; - if ((n->class & ~PHEAP) < nelem(classnames)) + if(n->class & PHEAP) s = ",heap"; + if((n->class & ~PHEAP) < nelem(classnames)) fmtprint(fp, " class(%s%s)", classnames[n->class&~PHEAP], s); else fmtprint(fp, " class(%d?%s)", n->class&~PHEAP, s); @@ -1107,8 +1120,29 @@ Jconv(Fmt *fp) if(n->funcdepth != 0) fmtprint(fp, " f(%d)", n->funcdepth); - if(n->noescape != 0) - fmtprint(fp, " ne(%d)", n->noescape); + switch(n->esc) { + case EscUnknown: + break; + case EscHeap: + fmtprint(fp, " esc(h)"); + break; + case EscScope: + fmtprint(fp, " esc(s)"); + break; + case EscNone: + fmtprint(fp, " esc(no)"); + break; + case EscNever: + if(!c) + fmtprint(fp, " esc(N)"); + break; + default: + fmtprint(fp, " esc(%d)", n->esc); + break; + } + + if(n->escloopdepth) + fmtprint(fp, " ld(%d)", n->escloopdepth); if(!c && n->typecheck != 0) fmtprint(fp, " tc(%d)", n->typecheck); @@ -1122,9 +1156,6 @@ Jconv(Fmt *fp) if(n->implicit != 0) fmtprint(fp, " implicit(%d)", n->implicit); - if(!c && n->pun != 0) - fmtprint(fp, " pun(%d)", n->pun); - if(!c && n->used != 0) fmtprint(fp, " used(%d)", n->used); return 0; @@ -1523,7 +1554,7 @@ Nconv(Fmt *fp) switch(n->op) { default: - if (fp->flags & FmtShort) + if(fp->flags & FmtShort) fmtprint(fp, "%O%hJ", n->op, n); else fmtprint(fp, "%O%J", n->op, n); @@ -1532,13 +1563,13 @@ Nconv(Fmt *fp) case ONAME: case ONONAME: if(n->sym == S) { - if (fp->flags & FmtShort) + if(fp->flags & FmtShort) fmtprint(fp, "%O%hJ", n->op, n); else fmtprint(fp, "%O%J", n->op, n); break; } - if (fp->flags & FmtShort) + if(fp->flags & FmtShort) fmtprint(fp, "%O-%S%hJ", n->op, n->sym, n); else fmtprint(fp, "%O-%S%J", n->op, n->sym, n); @@ -2754,8 +2785,7 @@ copyexpr(Node *n, Type *t, NodeList **init) { Node *a, *l; - l = nod(OXXX, N, N); - tempname(l, t); + l = temp(t); a = nod(OAS, l, n); typecheck(&a, Etop); walkexpr(&a, init); @@ -2806,10 +2836,12 @@ setmaxarg(Type *t) maxarg = w; } -/* unicode-aware case-insensitive strcmp */ +/* + * unicode-aware case-insensitive strcmp + */ static int -cistrcmp(char *p, char *q) +ucistrcmp(char *p, char *q) { Rune rp, rq; @@ -2851,7 +2883,7 @@ lookdot0(Sym *s, Type *t, Type **save, int ignorecase) c = 0; if(u->etype == TSTRUCT || u->etype == TINTER) { for(f=u->type; f!=T; f=f->down) - if(f->sym == s || (ignorecase && cistrcmp(f->sym->name, s->name) == 0)) { + if(f->sym == s || (ignorecase && ucistrcmp(f->sym->name, s->name) == 0)) { if(save) *save = f; c++; @@ -2860,7 +2892,7 @@ lookdot0(Sym *s, Type *t, Type **save, int ignorecase) u = methtype(t); if(u != T) { for(f=u->method; f!=T; f=f->down) - if(f->embedded == 0 && (f->sym == s || (ignorecase && cistrcmp(f->sym->name, s->name) == 0))) { + if(f->embedded == 0 && (f->sym == s || (ignorecase && ucistrcmp(f->sym->name, s->name) == 0))) { if(save) *save = f; c++; @@ -3176,7 +3208,7 @@ genwrapper(Type *rcvr, Type *method, Sym *newnam, int iface) int isddd; Val v; - if(debug['r']) + if(0 && debug['r']) print("genwrapper rcvrtype=%T method=%T newnam=%S\n", rcvr, method, newnam); @@ -3410,8 +3442,13 @@ list1(Node *n) if(n == nil) return nil; - if(n->op == OBLOCK && n->ninit == nil) - return n->list; + if(n->op == OBLOCK && n->ninit == nil) { + // Flatten list and steal storage. + // Poison pointer to catch errant uses. + l = n->list; + n->list = (NodeList*)1; + return l; + } l = mal(sizeof *l); l->n = n; l->end = l; @@ -3453,7 +3490,7 @@ listsort(NodeList** l, int(*f)(Node*, Node*)) listsort(&l1, f); listsort(&l2, f); - if ((*f)(l1->n, l2->n) < 0) { + if((*f)(l1->n, l2->n) < 0) { *l = l1; } else { *l = l2; @@ -3469,7 +3506,7 @@ listsort(NodeList** l, int(*f)(Node*, Node*)) // l1 is last one from l1 that is < l2 le = l1->next; // le is the rest of l1, first one that is >= l2 - if (le != nil) + if(le != nil) le->end = (*l)->end; (*l)->end = l1; // cut *l at l1 diff --git a/src/cmd/gc/swt.c b/src/cmd/gc/swt.c index c2968c44b..0381132d0 100644 --- a/src/cmd/gc/swt.c +++ b/src/cmd/gc/swt.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" enum @@ -513,8 +515,7 @@ exprswitch(Node *sw) exprname = N; cas = nil; if(arg != Strue && arg != Sfalse) { - exprname = nod(OXXX, N, N); - tempname(exprname, sw->ntest->type); + exprname = temp(sw->ntest->type); cas = list1(nod(OAS, exprname, sw->ntest)); typechecklist(cas, Etop); } @@ -671,20 +672,17 @@ typeswitch(Node *sw) * predeclare temporary variables * and the boolean var */ - facename = nod(OXXX, N, N); - tempname(facename, sw->ntest->right->type); + facename = temp(sw->ntest->right->type); a = nod(OAS, facename, sw->ntest->right); typecheck(&a, Etop); cas = list(cas, a); casebody(sw, facename); - boolname = nod(OXXX, N, N); - tempname(boolname, types[TBOOL]); + boolname = temp(types[TBOOL]); typecheck(&boolname, Erv); - hashname = nod(OXXX, N, N); - tempname(hashname, types[TUINT32]); + hashname = temp(types[TUINT32]); typecheck(&hashname, Erv); t = sw->ntest->right->type; @@ -865,9 +863,9 @@ typecheckswitch(Node *n) yyerror("case %+N in %T switch", ll->n, t); break; case Etype: // type switch - if(ll->n->op == OLITERAL && istype(ll->n->type, TNIL)) + if(ll->n->op == OLITERAL && istype(ll->n->type, TNIL)) { ; - else if(ll->n->op != OTYPE && ll->n->type != T) { + } else if(ll->n->op != OTYPE && ll->n->type != T) { yyerror("%#N is not a type", ll->n); // reset to original type ll->n = n->ntest->right; diff --git a/src/cmd/gc/typecheck.c b/src/cmd/gc/typecheck.c index 78cdb5bf2..b9c302ce8 100644 --- a/src/cmd/gc/typecheck.c +++ b/src/cmd/gc/typecheck.c @@ -10,6 +10,8 @@ * rewrites n->op to be more specific in some cases. */ +#include +#include #include "go.h" static void implicitstar(Node**); @@ -21,7 +23,6 @@ static void typecheckaste(int, Node*, int, Type*, NodeList*, char*); static Type* lookdot1(Sym *s, Type *t, Type *f, int); static int nokeys(NodeList*); static void typecheckcomplit(Node**); -static void addrescapes(Node*); static void typecheckas2(Node*); static void typecheckas(Node*); static void typecheckfunc(Node*); @@ -37,7 +38,7 @@ static NodeList* typecheckdefstack; /* * resolve ONONAME to definition, if any. */ -Node* +static Node* resolve(Node *n) { Node *r; @@ -152,7 +153,7 @@ typecheck(Node **np, int top) } if(n->typecheck == 2) { - yyerror("typechecking loop"); + yyerror("typechecking loop involving %#N", n); lineno = lno; return n; } @@ -337,7 +338,7 @@ reswitch: */ case OIND: ntop = Erv | Etype; - if(!(top & Eaddr)) + if(!(top & Eaddr)) // The *x in &*x is not an indirect. ntop |= Eindir; l = typecheck(&n->left, ntop); if((t = l->type) == T) @@ -535,7 +536,9 @@ reswitch: l = n->left; if((t = l->type) == T) goto error; - if(!(top & Eindir) && !n->etype) + // top&Eindir means this is &x in *&x. (or the arg to built-in print) + // n->etype means code generator flagged it as non-escaping. + if(debug['s'] && !(top & Eindir) && !n->etype) addrescapes(n->left); n->type = ptrto(t); goto ret; @@ -721,7 +724,7 @@ reswitch: } defaultlit(&n->right, t->type); r = n->right; - if((t = r->type) == T) + if(r->type == T) goto error; r = assignconv(r, l->type->type, "send"); // TODO: more aggressive @@ -1028,6 +1031,7 @@ reswitch: } n->left = args->n; n->right = args->next->n; + n->list = nil; n->type = types[TINT]; typecheck(&n->left, Erv); typecheck(&n->right, Erv); @@ -1038,7 +1042,7 @@ reswitch: // copy([]byte, string) if(isslice(n->left->type) && n->right->type->etype == TSTRING) { - if (n->left->type->type == types[TUINT8]) + if(n->left->type->type == types[TUINT8]) goto ret; yyerror("arguments to copy have different element types: %lT and string", n->left->type); goto error; @@ -1068,7 +1072,7 @@ reswitch: goto error; if((n->op = convertop(t, n->type, &why)) == 0) { yyerror("cannot convert %+N to type %T%s", n->left, n->type, why); - op = OCONV; + n->op = OCONV; } switch(n->op) { case OCONVNOP: @@ -1602,7 +1606,8 @@ lookdot(Node *n, Type *t, int dostrcmp) if(!eqtype(rcvr, tt)) { if(rcvr->etype == tptr && eqtype(rcvr->type, tt)) { checklvalue(n->left, "call pointer method on"); - addrescapes(n->left); + if(debug['s']) + addrescapes(n->left); n->left = nod(OADDR, n->left, N); n->left->implicit = 1; typecheck(&n->left, Etype|Erv); @@ -2097,6 +2102,7 @@ typecheckcomplit(Node **np) yyerror("implicit assignment of unexported field '%s' in %T literal", s->name, t); ll->n = assignconv(ll->n, f->type, "field value"); ll->n = nod(OKEY, newname(f->sym), ll->n); + ll->n->left->type = f; ll->n->left->typecheck = 1; f = f->down; } @@ -2126,14 +2132,15 @@ typecheckcomplit(Node **np) // before we do the lookup. if(s->pkg != localpkg) s = lookup(s->name); - l->left = newname(s); - l->left->typecheck = 1; f = lookdot1(s, t, t->type, 0); typecheck(&l->right, Erv); if(f == nil) { yyerror("unknown %T field '%s' in struct literal", t, s->name); continue; } + l->left = newname(s); + l->left->typecheck = 1; + l->left->type = f; s = f->sym; fielddup(newname(s), hash, nhash); l->right = assignconv(l->right, f->type, "field value"); @@ -2156,82 +2163,6 @@ error: lineno = lno; } -/* - * the address of n has been taken and might be used after - * the current function returns. mark any local vars - * as needing to move to the heap. - */ -static void -addrescapes(Node *n) -{ - char buf[100]; - switch(n->op) { - default: - // probably a type error already. - // dump("addrescapes", n); - break; - - case ONAME: - if(n->noescape) - break; - switch(n->class) { - case PPARAMREF: - addrescapes(n->defn); - break; - case PPARAM: - case PPARAMOUT: - // if func param, need separate temporary - // to hold heap pointer. - // the function type has already been checked - // (we're in the function body) - // so the param already has a valid xoffset. - - // expression to refer to stack copy - n->stackparam = nod(OPARAM, n, N); - n->stackparam->type = n->type; - n->stackparam->addable = 1; - if(n->xoffset == BADWIDTH) - fatal("addrescapes before param assignment"); - n->stackparam->xoffset = n->xoffset; - n->xoffset = 0; - // fallthrough - case PAUTO: - - n->class |= PHEAP; - n->addable = 0; - n->ullman = 2; - n->xoffset = 0; - - // create stack variable to hold pointer to heap - n->heapaddr = nod(ONAME, N, N); - n->heapaddr->type = ptrto(n->type); - snprint(buf, sizeof buf, "&%S", n->sym); - n->heapaddr->sym = lookup(buf); - n->heapaddr->class = PHEAP-1; // defer tempname to allocparams - n->heapaddr->ullman = 1; - n->curfn->dcl = list(n->curfn->dcl, n->heapaddr); - - break; - } - break; - - case OIND: - case ODOTPTR: - break; - - case ODOT: - case OINDEX: - // ODOTPTR has already been introduced, - // so these are the non-pointer ODOT and OINDEX. - // In &x[0], if x is a slice, then x does not - // escape--the pointer inside x does, but that - // is always a heap pointer anyway. - if(!isslice(n->left->type)) - addrescapes(n->left); - break; - } -} - /* * lvalue etc */ @@ -2462,7 +2393,6 @@ typecheckfunc(Node *n) { Type *t, *rcvr; -//dump("nname", n->nname); typecheck(&n->nname, Erv | Easgn); if((t = n->nname->type) == T) return; @@ -2514,7 +2444,7 @@ getforwtype(Node *n) { Node *f1, *f2; - for(f1=f2=n; ; n=n->ntype) { + for(f2=n; ; n=n->ntype) { if((n = resolve(n)) == N || n->op != OTYPE) return T; @@ -2772,6 +2702,7 @@ typecheckdef(Node *n) if(n->ntype != N) { typecheck(&n->ntype, Etype); n->type = n->ntype->type; + if(n->type == T) { n->diag = 1; goto ret; diff --git a/src/cmd/gc/unsafe.c b/src/cmd/gc/unsafe.c index d304077c8..6435492e0 100644 --- a/src/cmd/gc/unsafe.c +++ b/src/cmd/gc/unsafe.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" /* diff --git a/src/cmd/gc/walk.c b/src/cmd/gc/walk.c index 9cd4ee919..8a84956a6 100644 --- a/src/cmd/gc/walk.c +++ b/src/cmd/gc/walk.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include +#include #include "go.h" static Node* walkprint(Node*, NodeList**, int); @@ -11,7 +13,7 @@ static Node* makenewvar(Type*, NodeList**, Node**); static Node* ascompatee1(int, Node*, Node*, NodeList**); static NodeList* ascompatee(int, NodeList*, NodeList*, NodeList**); static NodeList* ascompatet(int, NodeList*, Type**, int, NodeList**); -static NodeList* ascompatte(int, int, Type**, NodeList*, int, NodeList**); +static NodeList* ascompatte(int, Node*, int, Type**, NodeList*, int, NodeList**); static Node* convas(Node*, NodeList**); static void heapmoves(void); static NodeList* paramstoheap(Type **argin, int out); @@ -287,7 +289,7 @@ walkstmt(Node **np) n->list = reorder3(ll); break; } - ll = ascompatte(n->op, 0, getoutarg(curfn->type), n->list, 1, &n->ninit); + ll = ascompatte(n->op, nil, 0, getoutarg(curfn->type), n->list, 1, &n->ninit); n->list = ll; break; @@ -378,14 +380,11 @@ walkexpr(Node **np, NodeList **init) fatal("missed typecheck"); } - t = T; - et = Txxx; - switch(n->op) { default: dump("walk", n); fatal("walkexpr: switch 1 unknown op %N", n); - goto ret; + break; case OTYPE: case ONONAME: @@ -482,7 +481,7 @@ walkexpr(Node **np, NodeList **init) goto ret; walkexpr(&n->left, init); walkexprlist(n->list, init); - ll = ascompatte(n->op, n->isddd, getinarg(t), n->list, 0, init); + ll = ascompatte(n->op, n, n->isddd, getinarg(t), n->list, 0, init); n->list = reorder1(ll); goto ret; @@ -499,7 +498,7 @@ walkexpr(Node **np, NodeList **init) walkexpr(&n->left, init); walkexprlist(n->list, init); - ll = ascompatte(n->op, n->isddd, getinarg(t), n->list, 0, init); + ll = ascompatte(n->op, n, n->isddd, getinarg(t), n->list, 0, init); n->list = reorder1(ll); goto ret; @@ -509,8 +508,8 @@ walkexpr(Node **np, NodeList **init) goto ret; walkexpr(&n->left, init); walkexprlist(n->list, init); - ll = ascompatte(n->op, 0, getthis(t), list1(n->left->left), 0, init); - lr = ascompatte(n->op, n->isddd, getinarg(t), n->list, 0, init); + ll = ascompatte(n->op, n, 0, getthis(t), list1(n->left->left), 0, init); + lr = ascompatte(n->op, n, n->isddd, getinarg(t), n->list, 0, init); ll = concat(ll, lr); n->left->left = N; ullmancalc(n->left); @@ -563,8 +562,7 @@ walkexpr(Node **np, NodeList **init) // and map index has an implicit one. lpost = nil; if(l->op == OINDEXMAP) { - var = nod(OXXX, N, N); - tempname(var, l->type); + var = temp(l->type); n->list->n = var; a = nod(OAS, l, var); typecheck(&a, Etop); @@ -572,8 +570,7 @@ walkexpr(Node **np, NodeList **init) } l = n->list->next->n; if(l->op == OINDEXMAP) { - var = nod(OXXX, N, N); - tempname(var, l->type); + var = temp(l->type); n->list->next->n = var; a = nod(OAS, l, var); typecheck(&a, Etop); @@ -975,7 +972,15 @@ walkexpr(Node **np, NodeList **init) goto ret; case ONEW: - n = callnew(n->type->type); + if(n->esc == EscNone && n->type->type->width < (1<<16)) { + r = temp(n->type->type); + *init = list(*init, nod(OAS, r, N)); // zero temp + r = nod(OADDR, r, N); + typecheck(&r, Erv); + n = r; + } else { + n = callnew(n->type->type); + } goto ret; case OCMPSTR: @@ -1156,8 +1161,7 @@ walkexpr(Node **np, NodeList **init) case OARRAYLIT: case OMAPLIT: case OSTRUCTLIT: - nvar = nod(OXXX, N, N); - tempname(nvar, n->type); + nvar = temp(n->type); anylit(0, n, nvar, init); n = nvar; goto ret; @@ -1186,8 +1190,7 @@ makenewvar(Type *t, NodeList **init, Node **nstar) { Node *nvar, *nas; - nvar = nod(OXXX, N, N); - tempname(nvar, t); + nvar = temp(t); nas = nod(OAS, nvar, callnew(t->type)); typecheck(&nas, Etop); walkexpr(&nas, init); @@ -1201,6 +1204,8 @@ makenewvar(Type *t, NodeList **init, Node **nstar) static Node* ascompatee1(int op, Node *l, Node *r, NodeList **init) { + USED(op); + return convas(nod(OAS, l, r), init); } @@ -1257,6 +1262,8 @@ ascompatet(int op, NodeList *nl, Type **nr, int fp, NodeList **init) int ucount; NodeList *nn, *mm; + USED(op); + /* * check assign type list to * a expression list. called in @@ -1279,8 +1286,7 @@ ascompatet(int op, NodeList *nl, Type **nr, int fp, NodeList **init) // deferred until all the return arguments // have been pulled from the output arguments if(fncall(l, r->type)) { - tmp = nod(OXXX, N, N); - tempname(tmp, r->type); + tmp = temp(r->type); typecheck(&tmp, Erv); a = nod(OAS, l, tmp); a = convas(a, init); @@ -1309,21 +1315,27 @@ ascompatet(int op, NodeList *nl, Type **nr, int fp, NodeList **init) * package all the arguments that match a ... T parameter into a []T. */ static NodeList* -mkdotargslice(NodeList *lr0, NodeList *nn, Type *l, int fp, NodeList **init) +mkdotargslice(NodeList *lr0, NodeList *nn, Type *l, int fp, NodeList **init, int esc) { Node *a, *n; Type *tslice; - + tslice = typ(TARRAY); tslice->type = l->type->type; tslice->bound = -1; - n = nod(OCOMPLIT, N, typenod(tslice)); - n->list = lr0; - typecheck(&n, Erv); - if(n->type == T) - fatal("mkdotargslice: typecheck failed"); - walkexpr(&n, init); + if(count(lr0) == 0) { + n = nodnil(); + n->type = tslice; + } else { + n = nod(OCOMPLIT, N, typenod(tslice)); + n->list = lr0; + n->esc = esc; + typecheck(&n, Erv); + if(n->type == T) + fatal("mkdotargslice: typecheck failed"); + walkexpr(&n, init); + } a = nod(OAS, nodarg(l, fp), n); nn = list(nn, convas(a, init)); @@ -1343,7 +1355,6 @@ dumptypes(Type **nl, char *what) fmtstrinit(&fmt); fmtprint(&fmt, "\t"); - l = structfirst(&savel, nl); first = 1; for(l = structfirst(&savel, nl); l != T; l = structnext(&savel)) { if(first) @@ -1387,8 +1398,9 @@ dumpnodetypes(NodeList *l, char *what) * func(expr-list) */ static NodeList* -ascompatte(int op, int isddd, Type **nl, NodeList *lr, int fp, NodeList **init) +ascompatte(int op, Node *call, int isddd, Type **nl, NodeList *lr, int fp, NodeList **init) { + int esc; Type *l, *ll; Node *r, *a; NodeList *nn, *lr0, *alist; @@ -1416,8 +1428,7 @@ ascompatte(int op, int isddd, Type **nl, NodeList *lr, int fp, NodeList **init) // copy into temporaries. alist = nil; for(l=structfirst(&savel, &r->type); l; l=structnext(&savel)) { - a = nod(OXXX, N, N); - tempname(a, l->type); + a = temp(l->type); alist = list(alist, a); } a = nod(OAS2, N, N); @@ -1452,7 +1463,10 @@ loop: // normal case -- make a slice of all // remaining arguments and pass it to // the ddd parameter. - nn = mkdotargslice(lr, nn, l, fp, init); + esc = EscUnknown; + if(call->right) + esc = call->right->esc; + nn = mkdotargslice(lr, nn, l, fp, init, esc); goto ret; } @@ -1720,7 +1734,7 @@ out: * then it is done first. otherwise must * make temp variables */ -NodeList* +static NodeList* reorder1(NodeList *all) { Node *f, *a, *n; @@ -1757,8 +1771,7 @@ reorder1(NodeList *all) } // make assignment of fncall to tempname - a = nod(OXXX, N, N); - tempname(a, n->right->type); + a = temp(n->right->type); a = nod(OAS, a, n->right); g = list(g, a); @@ -1846,7 +1859,7 @@ vmatch1(Node *l, Node *r) return 0; } -NodeList* +static NodeList* reorder3(NodeList *all) { Node *n1, *n2, *q; @@ -1861,8 +1874,7 @@ reorder3(NodeList *all) if(c2 > c1) { if(vmatch1(n1->left, n2->right)) { // delay assignment to n1->left - q = nod(OXXX, N, N); - tempname(q, n1->right->type); + q = temp(n1->right->type); q = nod(OAS, n1->left, q); n1->left = q->right; r = list(r, q); @@ -2125,8 +2137,7 @@ append(Node *n, NodeList **init) l = nil; - ns = nod(OXXX, N, N); // var s - tempname(ns, nsrc->type); + ns = temp(nsrc->type); l = list(l, nod(OAS, ns, nsrc)); // s = src na = nodintconst(argc); // const argc @@ -2143,8 +2154,7 @@ append(Node *n, NodeList **init) conv(na, types[TINT64])))); l = list(l, nx); - nn = nod(OXXX, N, N); // var n - tempname(nn, types[TINT]); + nn = temp(types[TINT]); l = list(l, nod(OAS, nn, nod(OLEN, ns, N))); // n = len(s) nx = nod(OSLICE, ns, nod(OKEY, N, nod(OADD, nn, na))); // ...s[:n+argc] diff --git a/src/cmd/godefs/main.c b/src/cmd/godefs/main.c index 6a8630179..38b2962fa 100644 --- a/src/cmd/godefs/main.c +++ b/src/cmd/godefs/main.c @@ -98,7 +98,10 @@ waitfor(int pid) int spawn(char *prog, char **argv) { - int pid = fork(); + int pid; + + USED(prog); + pid = fork(); if(pid < 0) sysfatal("fork: %r"); if(pid == 0) { diff --git a/src/cmd/godoc/Makefile b/src/cmd/godoc/Makefile index f40d71703..a8cf5d6aa 100644 --- a/src/cmd/godoc/Makefile +++ b/src/cmd/godoc/Makefile @@ -18,6 +18,7 @@ GOFILES=\ parser.go\ snippet.go\ spec.go\ + throttle.go\ utils.go\ zip.go\ diff --git a/src/cmd/godoc/appconfig.go b/src/cmd/godoc/appconfig.go index 9cbe7a443..052a9ebc8 100644 --- a/src/cmd/godoc/appconfig.go +++ b/src/cmd/godoc/appconfig.go @@ -11,9 +11,20 @@ package main const ( // zipFilename is the name of the .zip file // containing the file system served by godoc. - zipFilename = "go.zip" + zipFilename = "godoc.zip" // zipGoroot is the path of the goroot directory // in the .zip file. - zipGoroot = "/home/username/go" + zipGoroot = "/home/user/go" + + // If indexFilenames != "", the search index is + // initialized with the index stored in these + // files (otherwise it will be built at run-time, + // eventually). indexFilenames is a glob pattern; + // the specified files are concatenated in sorted + // order (by filename). + // app-engine limit: file sizes must be <= 10MB; + // use "split -b8m indexfile index.split." to get + // smaller files. + indexFilenames = "index.split.*" ) diff --git a/src/cmd/godoc/appinit.go b/src/cmd/godoc/appinit.go index 9b8987223..baba53fa6 100644 --- a/src/cmd/godoc/appinit.go +++ b/src/cmd/godoc/appinit.go @@ -23,11 +23,12 @@ // strings // never version of the strings package // ... // // app.yaml // app engine control file -// go.zip // zip file containing the file system to serve +// godoc.zip // .zip file containing the file system to serve // godoc // contains godoc sources // appinit.go // this file instead of godoc/main.go // appconfig.go // godoc for app engine configuration // ... // +// index.split.* // index file(s) containing the search index to serve // // To run app the engine emulator locally: // @@ -39,10 +40,11 @@ package main import ( - "alt/archive/zip" + "archive/zip" "http" "log" "os" + "path" ) func serveError(w http.ResponseWriter, r *http.Request, relpath string, err os.Error) { @@ -53,7 +55,16 @@ func serveError(w http.ResponseWriter, r *http.Request, relpath string, err os.E func init() { log.Println("initializing godoc ...") + log.Printf(".zip file = %s", zipFilename) + log.Printf(".zip GOROOT = %s", zipGoroot) + log.Printf("index files = %s", indexFilenames) + + // initialize flags for app engine *goroot = path.Join("/", zipGoroot) // fsHttp paths are relative to '/' + *indexEnabled = true + *indexFiles = indexFilenames + *maxResults = 0 // save space for now + *indexThrottle = 0.3 // in case *indexFiles is empty (and thus the indexer is run) // read .zip file and set up file systems const zipfile = zipFilename @@ -65,8 +76,8 @@ func init() { fsHttp = NewHttpZipFS(rc, *goroot) // initialize http handlers - initHandlers() readTemplates() + initHandlers() registerPublicHandlers(http.DefaultServeMux) // initialize default directory tree with corresponding timestamp. @@ -75,12 +86,10 @@ func init() { // initialize directory trees for user-defined file systems (-path flag). initDirTrees() - // create search index - // TODO(gri) Disabled for now as it takes too long. Find a solution for this. - /* - *indexEnabled = true + // initialize search index + if *indexEnabled { go indexer() - */ + } log.Println("godoc initialization complete") } diff --git a/src/cmd/godoc/codewalk.go b/src/cmd/godoc/codewalk.go index e2643e466..602aa43a8 100644 --- a/src/cmd/godoc/codewalk.go +++ b/src/cmd/godoc/codewalk.go @@ -13,7 +13,6 @@ package main import ( - "container/vector" "fmt" "http" "io" @@ -183,17 +182,17 @@ func codewalkDir(w http.ResponseWriter, r *http.Request, relpath, abspath string serveError(w, r, relpath, err) return } - var v vector.Vector + var v []interface{} for _, fi := range dir { name := fi.Name() if fi.IsDirectory() { - v.Push(&elem{name + "/", ""}) + v = append(v, &elem{name + "/", ""}) } else if strings.HasSuffix(name, ".xml") { cw, err := loadCodewalk(abspath + "/" + name) if err != nil { continue } - v.Push(&elem{name[0 : len(name)-len(".xml")], cw.Title}) + v = append(v, &elem{name[0 : len(name)-len(".xml")], cw.Title}) } } diff --git a/src/cmd/godoc/dirtrees.go b/src/cmd/godoc/dirtrees.go index aa590b363..7595ef96f 100644 --- a/src/cmd/godoc/dirtrees.go +++ b/src/cmd/godoc/dirtrees.go @@ -117,7 +117,7 @@ func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth i // though the directory doesn't contain any real package files - was bug) if synopses[0] == "" { // no "optimal" package synopsis yet; continue to collect synopses - file, err := parser.ParseFile(fset, filepath.Join(path, d.Name()), nil, + file, err := parseFile(fset, filepath.Join(path, d.Name()), parser.ParseComments|parser.PackageClauseOnly) if err == nil { hasPkgFiles = true diff --git a/src/cmd/godoc/doc.go b/src/cmd/godoc/doc.go index dc98b0eca..813527d28 100644 --- a/src/cmd/godoc/doc.go +++ b/src/cmd/godoc/doc.go @@ -50,6 +50,17 @@ The flags are: -index enable identifier and full text search index (no search box is shown if -index is not set) + -index_files="" + glob pattern specifying index files; if not empty, + the index is read from these files in sorted order + -index_throttle=0.75 + index throttle value; a value of 0 means no time is allocated + to the indexer (the indexer will never finish), a value of 1.0 + means that index creation is running at full throttle (other + goroutines may get no time while the index is built) + -write_index=false + write index to a file; the file name must be specified with + -index_files -maxresults=10000 maximum number of full text search results shown (no full text index is built if maxresults <= 0) diff --git a/src/cmd/godoc/godoc.go b/src/cmd/godoc/godoc.go index b8a839404..6b646a1a6 100644 --- a/src/cmd/godoc/godoc.go +++ b/src/cmd/godoc/godoc.go @@ -64,7 +64,10 @@ var ( // search index indexEnabled = flag.Bool("index", false, "enable search index") - maxResults = flag.Int("maxresults", 10000, "maximum number of full text search results shown") + indexFiles = flag.String("index_files", "", "glob pattern specifying index files;"+ + "if not empty, the index is read from these files in sorted order") + maxResults = flag.Int("maxresults", 10000, "maximum number of full text search results shown") + indexThrottle = flag.Float64("index_throttle", 0.75, "index throttle value; 0.0 = no time allocated, 1.0 = full throttle") // file system mapping fs FileSystem // the underlying file system for godoc @@ -403,8 +406,8 @@ var infoKinds = [nKinds]string{ Use: "use", } -func infoKind_htmlFunc(kind SpotKind) string { - return infoKinds[kind] // infoKind entries are html-escaped +func infoKind_htmlFunc(info SpotInfo) string { + return infoKinds[info.Kind()] // infoKind entries are html-escaped } func infoLineFunc(info SpotInfo) int { @@ -540,7 +543,19 @@ func readTemplate(name string) *template.Template { path = defaultpath } } - return template.Must(template.New(name).Funcs(fmap).ParseFile(path)) + + // use underlying file system fs to read the template file + // (cannot use template ParseFile functions directly) + data, err := fs.ReadFile(path) + if err != nil { + log.Fatal("readTemplate: ", err) + } + // be explicit with errors (for app engine use) + t, err := template.New(name).Funcs(fmap).Parse(string(data)) + if err != nil { + log.Fatal("readTemplate: ", err) + } + return t } var ( @@ -1049,9 +1064,7 @@ func lookup(query string) (result SearchResult) { // is the result accurate? if *indexEnabled { if _, ts := fsModified.get(); timestamp < ts { - // The index is older than the latest file system change - // under godoc's observation. Indexing may be in progress - // or start shortly (see indexer()). + // The index is older than the latest file system change under godoc's observation. result.Alert = "Indexing in progress: result may be inaccurate" } } else { @@ -1128,26 +1141,61 @@ func fsDirnames() <-chan string { return c } +func readIndex(filenames string) os.Error { + matches, err := filepath.Glob(filenames) + if err != nil { + return err + } + sort.Strings(matches) // make sure files are in the right order + files := make([]io.Reader, 0, len(matches)) + for _, filename := range matches { + f, err := os.Open(filename) + if err != nil { + return err + } + defer f.Close() + files = append(files, f) + } + x := new(Index) + if err := x.Read(io.MultiReader(files...)); err != nil { + return err + } + searchIndex.set(x) + return nil +} + +func updateIndex() { + if *verbose { + log.Printf("updating index...") + } + start := time.Nanoseconds() + index := NewIndex(fsDirnames(), *maxResults > 0, *indexThrottle) + stop := time.Nanoseconds() + searchIndex.set(index) + if *verbose { + secs := float64((stop-start)/1e6) / 1e3 + stats := index.Stats() + log.Printf("index updated (%gs, %d bytes of source, %d files, %d lines, %d unique words, %d spots)", + secs, stats.Bytes, stats.Files, stats.Lines, stats.Words, stats.Spots) + } + log.Printf("before GC: bytes = %d footprint = %d", runtime.MemStats.HeapAlloc, runtime.MemStats.Sys) + runtime.GC() + log.Printf("after GC: bytes = %d footprint = %d", runtime.MemStats.HeapAlloc, runtime.MemStats.Sys) +} + func indexer() { + // initialize the index from disk if possible + if *indexFiles != "" { + if err := readIndex(*indexFiles); err != nil { + log.Printf("error reading index: %s", err) + } + } + + // repeatedly update the index when it goes out of date for { if !indexUpToDate() { // index possibly out of date - make a new one - if *verbose { - log.Printf("updating index...") - } - start := time.Nanoseconds() - index := NewIndex(fsDirnames(), *maxResults > 0) - stop := time.Nanoseconds() - searchIndex.set(index) - if *verbose { - secs := float64((stop-start)/1e6) / 1e3 - stats := index.Stats() - log.Printf("index updated (%gs, %d bytes of source, %d files, %d lines, %d unique words, %d spots)", - secs, stats.Bytes, stats.Files, stats.Lines, stats.Words, stats.Spots) - } - log.Printf("before GC: bytes = %d footprint = %d", runtime.MemStats.HeapAlloc, runtime.MemStats.Sys) - runtime.GC() - log.Printf("after GC: bytes = %d footprint = %d", runtime.MemStats.HeapAlloc, runtime.MemStats.Sys) + updateIndex() } var delay int64 = 60 * 1e9 // by default, try every 60s if *testDir != "" { diff --git a/src/cmd/godoc/index.go b/src/cmd/godoc/index.go index 9b4f31514..b99363491 100644 --- a/src/cmd/godoc/index.go +++ b/src/cmd/godoc/index.go @@ -7,7 +7,7 @@ // // Algorithm for identifier index: // - traverse all .go files of the file tree specified by root -// - for each word (identifier) encountered, collect all occurrences (spots) +// - for each identifier (word) encountered, collect all occurrences (spots) // into a list; this produces a list of spots for each word // - reduce the lists: from a list of spots to a list of FileRuns, // and from a list of FileRuns into a list of PakRuns @@ -39,12 +39,13 @@ package main import ( "bytes" - "container/vector" "go/ast" "go/parser" "go/token" "go/scanner" + "gob" "index/suffixarray" + "io" "os" "path/filepath" "regexp" @@ -52,48 +53,62 @@ import ( "strings" ) +// ---------------------------------------------------------------------------- +// InterfaceSlice is a helper type for sorting interface +// slices according to some slice-specific sort criteria. + +type Comparer func(x, y interface{}) bool + +type InterfaceSlice struct { + slice []interface{} + less Comparer +} + +func (p *InterfaceSlice) Len() int { return len(p.slice) } +func (p *InterfaceSlice) Less(i, j int) bool { return p.less(p.slice[i], p.slice[j]) } +func (p *InterfaceSlice) Swap(i, j int) { p.slice[i], p.slice[j] = p.slice[j], p.slice[i] } + // ---------------------------------------------------------------------------- // RunList -// A RunList is a vector of entries that can be sorted according to some +// A RunList is a list of entries that can be sorted according to some // criteria. A RunList may be compressed by grouping "runs" of entries // which are equal (according to the sort critera) into a new RunList of // runs. For instance, a RunList containing pairs (x, y) may be compressed // into a RunList containing pair runs (x, {y}) where each run consists of // a list of y's with the same x. -type RunList struct { - vector.Vector - less func(x, y interface{}) bool -} - -func (h *RunList) Less(i, j int) bool { return h.less(h.At(i), h.At(j)) } +type RunList []interface{} -func (h *RunList) sort(less func(x, y interface{}) bool) { - h.less = less - sort.Sort(h) +func (h RunList) sort(less Comparer) { + sort.Sort(&InterfaceSlice{h, less}) } // Compress entries which are the same according to a sort criteria // (specified by less) into "runs". -func (h *RunList) reduce(less func(x, y interface{}) bool, newRun func(h *RunList, i, j int) interface{}) *RunList { +func (h RunList) reduce(less Comparer, newRun func(h RunList) interface{}) RunList { + if len(h) == 0 { + return nil + } + // len(h) > 0 + // create runs of entries with equal values h.sort(less) // for each run, make a new run object and collect them in a new RunList var hh RunList - i := 0 - for j := 0; j < h.Len(); j++ { - if less(h.At(i), h.At(j)) { - hh.Push(newRun(h, i, j)) - i = j // start a new run + i, x := 0, h[0] + for j, y := range h { + if less(x, y) { + hh = append(hh, newRun(h[i:j])) + i, x = j, h[j] // start a new run } } // add final run, if any - if i < h.Len() { - hh.Push(newRun(h, i, h.Len())) + if i < len(h) { + hh = append(hh, newRun(h[i:])) } - return &hh + return hh } // ---------------------------------------------------------------------------- @@ -164,30 +179,25 @@ func (x SpotInfo) IsIndex() bool { return x&1 != 0 } const removeDuplicates = true // A KindRun is a run of SpotInfos of the same kind in a given file. -type KindRun struct { - Kind SpotKind - Infos []SpotInfo -} +// The kind (3 bits) is stored in each SpotInfo element; to find the +// kind of a KindRun, look at any of it's elements. +type KindRun []SpotInfo // KindRuns are sorted by line number or index. Since the isIndex bit // is always the same for all infos in one list we can compare lori's. -func (f *KindRun) Len() int { return len(f.Infos) } -func (f *KindRun) Less(i, j int) bool { return f.Infos[i].Lori() < f.Infos[j].Lori() } -func (f *KindRun) Swap(i, j int) { f.Infos[i], f.Infos[j] = f.Infos[j], f.Infos[i] } +func (k KindRun) Len() int { return len(k) } +func (k KindRun) Less(i, j int) bool { return k[i].Lori() < k[j].Lori() } +func (k KindRun) Swap(i, j int) { k[i], k[j] = k[j], k[i] } // FileRun contents are sorted by Kind for the reduction into KindRuns. func lessKind(x, y interface{}) bool { return x.(SpotInfo).Kind() < y.(SpotInfo).Kind() } -// newKindRun allocates a new KindRun from the SpotInfo run [i, j) in h. -func newKindRun(h *RunList, i, j int) interface{} { - kind := h.At(i).(SpotInfo).Kind() - infos := make([]SpotInfo, j-i) - k := 0 - for ; i < j; i++ { - infos[k] = h.At(i).(SpotInfo) - k++ +// newKindRun allocates a new KindRun from the SpotInfo run h. +func newKindRun(h RunList) interface{} { + run := make(KindRun, len(h)) + for i, x := range h { + run[i] = x.(SpotInfo) } - run := &KindRun{kind, infos} // Spots were sorted by file and kind to create this run. // Within this run, sort them by line number or index. @@ -199,15 +209,15 @@ func newKindRun(h *RunList, i, j int) interface{} { // bit is always the same for all infos in one // list we can simply compare the entire info. k := 0 - var prev SpotInfo - for i, x := range infos { - if x != prev || i == 0 { - infos[k] = x + prev := SpotInfo(1<<32 - 1) // an unlikely value + for _, x := range run { + if x != prev { + run[k] = x k++ prev = x } } - run.Infos = infos[0:k] + run = run[0:k] } return run @@ -229,8 +239,13 @@ func (p *Pak) less(q *Pak) bool { // A File describes a Go file. type File struct { - Path string // complete file name - Pak Pak // the package to which the file belongs + Name string // directory-local file name + Pak *Pak // the package to which the file belongs +} + +// Path returns the file path of f. +func (f *File) Path() string { + return filepath.Join(f.Pak.Path, f.Name) } // A Spot describes a single occurrence of a word. @@ -242,30 +257,34 @@ type Spot struct { // A FileRun is a list of KindRuns belonging to the same file. type FileRun struct { File *File - Groups []*KindRun + Groups []KindRun } -// Spots are sorted by path for the reduction into FileRuns. -func lessSpot(x, y interface{}) bool { return x.(Spot).File.Path < y.(Spot).File.Path } +// Spots are sorted by file path for the reduction into FileRuns. +func lessSpot(x, y interface{}) bool { + fx := x.(Spot).File + fy := y.(Spot).File + // same as "return fx.Path() < fy.Path()" but w/o computing the file path first + px := fx.Pak.Path + py := fy.Pak.Path + return px < py || px == py && fx.Name < fy.Name +} -// newFileRun allocates a new FileRun from the Spot run [i, j) in h. -func newFileRun(h0 *RunList, i, j int) interface{} { - file := h0.At(i).(Spot).File +// newFileRun allocates a new FileRun from the Spot run h. +func newFileRun(h RunList) interface{} { + file := h[0].(Spot).File // reduce the list of Spots into a list of KindRuns - var h1 RunList - h1.Vector.Resize(j-i, 0) - k := 0 - for ; i < j; i++ { - h1.Set(k, h0.At(i).(Spot).Info) - k++ + h1 := make(RunList, len(h)) + for i, x := range h { + h1[i] = x.(Spot).Info } h2 := h1.reduce(lessKind, newKindRun) // create the FileRun - groups := make([]*KindRun, h2.Len()) - for i := 0; i < h2.Len(); i++ { - groups[i] = h2.At(i).(*KindRun) + groups := make([]KindRun, len(h2)) + for i, x := range h2 { + groups[i] = x.(KindRun) } return &FileRun{file, groups} } @@ -275,28 +294,26 @@ func newFileRun(h0 *RunList, i, j int) interface{} { // A PakRun describes a run of *FileRuns of a package. type PakRun struct { - Pak Pak + Pak *Pak Files []*FileRun } // Sorting support for files within a PakRun. func (p *PakRun) Len() int { return len(p.Files) } -func (p *PakRun) Less(i, j int) bool { return p.Files[i].File.Path < p.Files[j].File.Path } +func (p *PakRun) Less(i, j int) bool { return p.Files[i].File.Name < p.Files[j].File.Name } func (p *PakRun) Swap(i, j int) { p.Files[i], p.Files[j] = p.Files[j], p.Files[i] } // FileRuns are sorted by package for the reduction into PakRuns. func lessFileRun(x, y interface{}) bool { - return x.(*FileRun).File.Pak.less(&y.(*FileRun).File.Pak) + return x.(*FileRun).File.Pak.less(y.(*FileRun).File.Pak) } -// newPakRun allocates a new PakRun from the *FileRun run [i, j) in h. -func newPakRun(h *RunList, i, j int) interface{} { - pak := h.At(i).(*FileRun).File.Pak - files := make([]*FileRun, j-i) - k := 0 - for ; i < j; i++ { - files[k] = h.At(i).(*FileRun) - k++ +// newPakRun allocates a new PakRun from the *FileRun run h. +func newPakRun(h RunList) interface{} { + pak := h[0].(*FileRun).File.Pak + files := make([]*FileRun, len(h)) + for i, x := range h { + files[i] = x.(*FileRun) } run := &PakRun{pak, files} sort.Sort(run) // files were sorted by package; sort them by file now @@ -310,9 +327,9 @@ func newPakRun(h *RunList, i, j int) interface{} { type HitList []*PakRun // PakRuns are sorted by package. -func lessPakRun(x, y interface{}) bool { return x.(*PakRun).Pak.less(&y.(*PakRun).Pak) } +func lessPakRun(x, y interface{}) bool { return x.(*PakRun).Pak.less(y.(*PakRun).Pak) } -func reduce(h0 *RunList) HitList { +func reduce(h0 RunList) HitList { // reduce a list of Spots into a list of FileRuns h1 := h0.reduce(lessSpot, newFileRun) // reduce a list of FileRuns into a list of PakRuns @@ -320,28 +337,18 @@ func reduce(h0 *RunList) HitList { // sort the list of PakRuns by package h2.sort(lessPakRun) // create a HitList - h := make(HitList, h2.Len()) - for i := 0; i < h2.Len(); i++ { - h[i] = h2.At(i).(*PakRun) + h := make(HitList, len(h2)) + for i, p := range h2 { + h[i] = p.(*PakRun) } return h } func (h HitList) filter(pakname string) HitList { - // determine number of matching packages (most of the time just one) - n := 0 + var hh HitList for _, p := range h { if p.Pak.Name == pakname { - n++ - } - } - // create filtered HitList - hh := make(HitList, n) - i := 0 - for _, p := range h { - if p.Pak.Name == pakname { - hh[i] = p - i++ + hh = append(hh, p) } } return hh @@ -365,34 +372,27 @@ type AltWords struct { // wordPairs are sorted by their canonical spelling. func lessWordPair(x, y interface{}) bool { return x.(*wordPair).canon < y.(*wordPair).canon } -// newAltWords allocates a new AltWords from the *wordPair run [i, j) in h. -func newAltWords(h *RunList, i, j int) interface{} { - canon := h.At(i).(*wordPair).canon - alts := make([]string, j-i) - k := 0 - for ; i < j; i++ { - alts[k] = h.At(i).(*wordPair).alt - k++ +// newAltWords allocates a new AltWords from the *wordPair run h. +func newAltWords(h RunList) interface{} { + canon := h[0].(*wordPair).canon + alts := make([]string, len(h)) + for i, x := range h { + alts[i] = x.(*wordPair).alt } return &AltWords{canon, alts} } func (a *AltWords) filter(s string) *AltWords { - if len(a.Alts) == 1 && a.Alts[0] == s { - // there are no different alternatives - return nil - } - - // make a new AltWords with the current spelling removed - alts := make([]string, len(a.Alts)) - i := 0 + var alts []string for _, w := range a.Alts { if w != s { - alts[i] = w - i++ + alts = append(alts, w) } } - return &AltWords{a.Canon, alts[0:i]} + if len(alts) > 0 { + return &AltWords{a.Canon, alts} + } + return nil } // ---------------------------------------------------------------------------- @@ -423,17 +423,32 @@ type Statistics struct { type Indexer struct { fset *token.FileSet // file set for all indexed files sources bytes.Buffer // concatenated sources + packages map[string]*Pak // map of canonicalized *Paks words map[string]*IndexResult // RunLists of Spots - snippets vector.Vector // vector of *Snippets, indexed by snippet indices + snippets []*Snippet // indices are stored in SpotInfos current *token.File // last file added to file set file *File // AST for current file decl ast.Decl // AST for current decl stats Statistics } +func (x *Indexer) lookupPackage(path, name string) *Pak { + // In the source directory tree, more than one package may + // live in the same directory. For the packages map, construct + // a key that includes both the directory path and the package + // name. + key := path + ":" + name + pak := x.packages[key] + if pak == nil { + pak = &Pak{path, name} + x.packages[key] = pak + } + return pak +} + func (x *Indexer) addSnippet(s *Snippet) int { - index := x.snippets.Len() - x.snippets.Push(s) + index := len(x.snippets) + x.snippets = append(x.snippets, s) return index } @@ -454,12 +469,12 @@ func (x *Indexer) visitIdent(kind SpotKind, id *ast.Ident) { if kind == Use || x.decl == nil { // not a declaration or no snippet required info := makeSpotInfo(kind, x.current.Line(id.Pos()), false) - lists.Others.Push(Spot{x.file, info}) + lists.Others = append(lists.Others, Spot{x.file, info}) } else { // a declaration with snippet index := x.addSnippet(NewSnippet(x.fset, x.decl, id)) info := makeSpotInfo(kind, index, true) - lists.Decls.Push(Spot{x.file, info}) + lists.Decls = append(lists.Decls, Spot{x.file, info}) } x.stats.Spots++ @@ -713,9 +728,8 @@ func (x *Indexer) visitFile(dirname string, f FileInfo, fulltextIndex bool) { if fast != nil { // we've got a Go file to index x.current = file - dir, _ := filepath.Split(filename) - pak := Pak{dir, fast.Name.Name} - x.file = &File{filename, pak} + pak := x.lookupPackage(dirname, fast.Name.Name) + x.file = &File{f.Name(), pak} ast.Walk(x, fast) } @@ -747,12 +761,15 @@ func canonical(w string) string { return strings.ToLower(w) } // NewIndex creates a new index for the .go files // in the directories given by dirnames. // -func NewIndex(dirnames <-chan string, fulltextIndex bool) *Index { +func NewIndex(dirnames <-chan string, fulltextIndex bool, throttle float64) *Index { var x Indexer + th := NewThrottle(throttle, 0.1e9) // run at least 0.1s at a time // initialize Indexer + // (use some reasonably sized maps to start) x.fset = token.NewFileSet() - x.words = make(map[string]*IndexResult) + x.packages = make(map[string]*Pak, 256) + x.words = make(map[string]*IndexResult, 8192) // index all files in the directories given by dirnames for dirname := range dirnames { @@ -764,6 +781,7 @@ func NewIndex(dirnames <-chan string, fulltextIndex bool) *Index { if !f.IsDirectory() { x.visitFile(dirname, f, fulltextIndex) } + th.Throttle() } } @@ -782,13 +800,14 @@ func NewIndex(dirnames <-chan string, fulltextIndex bool) *Index { words := make(map[string]*LookupResult) var wlist RunList for w, h := range x.words { - decls := reduce(&h.Decls) - others := reduce(&h.Others) + decls := reduce(h.Decls) + others := reduce(h.Others) words[w] = &LookupResult{ Decls: decls, Others: others, } - wlist.Push(&wordPair{canonical(w), w}) + wlist = append(wlist, &wordPair{canonical(w), w}) + th.Throttle() } x.stats.Words = len(words) @@ -798,24 +817,49 @@ func NewIndex(dirnames <-chan string, fulltextIndex bool) *Index { // convert alist into a map of alternative spellings alts := make(map[string]*AltWords) - for i := 0; i < alist.Len(); i++ { - a := alist.At(i).(*AltWords) + for i := 0; i < len(alist); i++ { + a := alist[i].(*AltWords) alts[a.Canon] = a } - // convert snippet vector into a list - snippets := make([]*Snippet, x.snippets.Len()) - for i := 0; i < x.snippets.Len(); i++ { - snippets[i] = x.snippets.At(i).(*Snippet) - } - // create text index var suffixes *suffixarray.Index if fulltextIndex { suffixes = suffixarray.New(x.sources.Bytes()) } - return &Index{x.fset, suffixes, words, alts, snippets, x.stats} + return &Index{x.fset, suffixes, words, alts, x.snippets, x.stats} +} + +type FileIndex struct { + Words map[string]*LookupResult + Alts map[string]*AltWords + Snippets []*Snippet +} + +// Write writes the index x to w. +func (x *Index) Write(w io.Writer) os.Error { + if x.suffixes != nil { + panic("no support for writing full text index yet") + } + fx := FileIndex{ + x.words, + x.alts, + x.snippets, + } + return gob.NewEncoder(w).Encode(fx) +} + +// Read reads the index from r into x; x must not be nil. +func (x *Index) Read(r io.Reader) os.Error { + var fx FileIndex + if err := gob.NewDecoder(r).Decode(&fx); err != nil { + return err + } + x.words = fx.Words + x.alts = fx.Alts + x.snippets = fx.Snippets + return nil } // Stats() returns index statistics. diff --git a/src/cmd/godoc/main.go b/src/cmd/godoc/main.go index 89b12b9ac..74d3111ff 100644 --- a/src/cmd/godoc/main.go +++ b/src/cmd/godoc/main.go @@ -54,6 +54,9 @@ var ( // (with e.g.: zip -r go.zip $GOROOT -i \*.go -i \*.html -i \*.css -i \*.js -i \*.txt -i \*.c -i \*.h -i \*.s -i \*.png -i \*.jpg -i \*.sh -i favicon.ico) zipfile = flag.String("zip", "", "zip file providing the file system to serve; disabled if empty") + // file-based index + writeIndex = flag.Bool("write_index", false, "write index to a file; the file name must be specified with -index_files") + // periodic sync syncCmd = flag.String("sync", "", "sync command; disabled if empty") syncMin = flag.Int("sync_minutes", 0, "sync interval in minutes; disabled if <= 0") @@ -221,8 +224,8 @@ func main() { flag.Usage = usage flag.Parse() - // Check usage: either server and no args, or command line and args - if (*httpAddr != "") != (flag.NArg() == 0) { + // Check usage: either server and no args, command line and args, or index creation mode + if (*httpAddr != "") != (flag.NArg() == 0) && !*writeIndex { usage() } @@ -245,13 +248,48 @@ func main() { if err != nil { log.Fatalf("%s: %s\n", *zipfile, err) } + defer rc.Close() // be nice (e.g., -writeIndex mode) *goroot = path.Join("/", *goroot) // fsHttp paths are relative to '/' fs = NewZipFS(rc) fsHttp = NewHttpZipFS(rc, *goroot) } - initHandlers() readTemplates() + initHandlers() + + if (*indexEnabled || *writeIndex) && *indexFiles != "" && *maxResults > 0 { + log.Println("warning: no support for full-text index yet (setting -maxresults to 0)") + *maxResults = 0 + } + + if *writeIndex { + // Write search index and exit. + if *indexFiles == "" { + log.Fatal("no index file specified") + } + + log.Println("initialize file systems") + *verbose = true // want to see what happens + initFSTree() + initDirTrees() + + *indexThrottle = 1 + updateIndex() + + log.Println("writing index file", *indexFiles) + f, err := os.Create(*indexFiles) + if err != nil { + log.Fatal(err) + } + index, _ := searchIndex.get() + err = index.(*Index).Write(f) + if err != nil { + log.Fatal(err) + } + + log.Println("done") + return + } if *httpAddr != "" { // HTTP server mode. @@ -304,7 +342,7 @@ func main() { }() } - // Start indexing goroutine. + // Initialize search index. if *indexEnabled { go indexer() } diff --git a/src/cmd/godoc/parser.go b/src/cmd/godoc/parser.go index da4b3853c..cc1780a4b 100644 --- a/src/cmd/godoc/parser.go +++ b/src/cmd/godoc/parser.go @@ -17,18 +17,18 @@ import ( "path/filepath" ) +func parseFile(fset *token.FileSet, filename string, mode uint) (*ast.File, os.Error) { + src, err := fs.ReadFile(filename) + if err != nil { + return nil, err + } + return parser.ParseFile(fset, filename, src, mode) +} + func parseFiles(fset *token.FileSet, filenames []string) (pkgs map[string]*ast.Package, first os.Error) { pkgs = make(map[string]*ast.Package) for _, filename := range filenames { - src, err := fs.ReadFile(filename) - if err != nil { - if first == nil { - first = err - } - continue - } - - file, err := parser.ParseFile(fset, filename, src, parser.ParseComments) + file, err := parseFile(fset, filename, parser.ParseComments) if err != nil { if first == nil { first = err diff --git a/src/cmd/godoc/snippet.go b/src/cmd/godoc/snippet.go old mode 100755 new mode 100644 diff --git a/src/cmd/godoc/throttle.go b/src/cmd/godoc/throttle.go new file mode 100644 index 000000000..193492802 --- /dev/null +++ b/src/cmd/godoc/throttle.go @@ -0,0 +1,88 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "time" + +// A Throttle permits throttling of a goroutine by +// calling the Throttle method repeatedly. +// +type Throttle struct { + f float64 // f = (1-r)/r for 0 < r < 1 + tm int64 // minimum run time slice; >= 0 + tr int64 // accumulated time running + ts int64 // accumulated time stopped + tt int64 // earliest throttle time (= time Throttle returned + tm) +} + +// NewThrottle creates a new Throttle with a throttle value r and +// a minimum allocated run time slice of tm nanoseconds: +// +// r == 0: "empty" throttle; the goroutine is always sleeping +// r == 1: full throttle; the goroutine is never sleeping +// +// A value of r == 0.6 throttles a goroutine such that it runs +// approx. 60% of the time, and sleeps approx. 40% of the time. +// Values of r < 0 or r > 1 are clamped down to values between 0 and 1. +// Values of tm < 0 are set to 0. +// +func NewThrottle(r float64, tm int64) *Throttle { + var f float64 + switch { + case r <= 0: + f = -1 // indicates always sleep + case r >= 1: + f = 0 // assume r == 1 (never sleep) + default: + // 0 < r < 1 + f = (1 - r) / r + } + if tm < 0 { + tm = 0 + } + return &Throttle{f: f, tm: tm, tt: time.Nanoseconds() + tm} +} + +// Throttle calls time.Sleep such that over time the ratio tr/ts between +// accumulated run (tr) and sleep times (ts) approximates the value 1/(1-r) +// where r is the throttle value. Throttle returns immediately (w/o sleeping) +// if less than tm ns have passed since the last call to Throttle. +// +func (p *Throttle) Throttle() { + if p.f < 0 { + select {} // always sleep + } + + t0 := time.Nanoseconds() + if t0 < p.tt { + return // keep running (minimum time slice not exhausted yet) + } + + // accumulate running time + p.tr += t0 - (p.tt - p.tm) + + // compute sleep time + // Over time we want: + // + // tr/ts = r/(1-r) + // + // Thus: + // + // ts = tr*f with f = (1-r)/r + // + // After some incremental run time δr added to the total run time + // tr, the incremental sleep-time δs to get to the same ratio again + // after waking up from time.Sleep is: + if δs := int64(float64(p.tr)*p.f) - p.ts; δs > 0 { + time.Sleep(δs) + } + + // accumulate (actual) sleep time + t1 := time.Nanoseconds() + p.ts += t1 - t0 + + // set earliest next throttle time + p.tt = t1 + p.tm +} diff --git a/src/cmd/godoc/zip.go b/src/cmd/godoc/zip.go index 27dc142f5..46d7112e5 100644 --- a/src/cmd/godoc/zip.go +++ b/src/cmd/godoc/zip.go @@ -183,9 +183,10 @@ func (z zipList) lookup(name string) (index int, exact bool) { i := sort.Search(len(z), func(i int) bool { return name <= z[i].Name }) - if i < 0 { + if i >= len(z) { return -1, false } + // 0 <= i < len(z) if z[i].Name == name { return i, true } @@ -196,9 +197,10 @@ func (z zipList) lookup(name string) (index int, exact bool) { j := sort.Search(len(z), func(i int) bool { return name <= z[i].Name }) - if j < 0 { + if j >= len(z) { return -1, false } + // 0 <= j < len(z) if strings.HasPrefix(z[j].Name, name) { return i + j, false } diff --git a/src/cmd/gofix/Makefile b/src/cmd/gofix/Makefile index 22033d7f8..d1f3ac605 100644 --- a/src/cmd/gofix/Makefile +++ b/src/cmd/gofix/Makefile @@ -14,6 +14,7 @@ GOFILES=\ httpserver.go\ main.go\ netdial.go\ + netudpgroup.go\ oserrorstring.go\ osopen.go\ procattr.go\ diff --git a/src/cmd/gofix/netudpgroup.go b/src/cmd/gofix/netudpgroup.go new file mode 100644 index 000000000..347452d43 --- /dev/null +++ b/src/cmd/gofix/netudpgroup.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/ast" +) + +var netudpgroupFix = fix{ + "netudpgroup", + netudpgroup, + `Adapt 1-argument calls of net.(*UDPConn).JoinGroup, LeaveGroup to use 2-argument form. + +http://codereview.appspot.com/4815074 +`, +} + +func init() { + register(netudpgroupFix) +} + +func netudpgroup(f *ast.File) bool { + if !imports(f, "net") { + return false + } + + fixed := false + for _, d := range f.Decls { + fd, ok := d.(*ast.FuncDecl) + if !ok { + continue + } + walk(fd.Body, func(n interface{}) { + ce, ok := n.(*ast.CallExpr) + if !ok { + return + } + se, ok := ce.Fun.(*ast.SelectorExpr) + if !ok || len(ce.Args) != 1 { + return + } + switch se.Sel.String() { + case "JoinGroup", "LeaveGroup": + // c.JoinGroup(a) -> c.JoinGroup(nil, a) + // c.LeaveGroup(a) -> c.LeaveGroup(nil, a) + arg := ce.Args[0] + ce.Args = make([]ast.Expr, 2) + ce.Args[0] = ast.NewIdent("nil") + ce.Args[1] = arg + fixed = true + } + }) + } + return fixed +} diff --git a/src/cmd/gofix/netudpgroup_test.go b/src/cmd/gofix/netudpgroup_test.go new file mode 100644 index 000000000..b3b5816da --- /dev/null +++ b/src/cmd/gofix/netudpgroup_test.go @@ -0,0 +1,33 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func init() { + addTestCases(netudpgroupTests) +} + +var netudpgroupTests = []testCase{ + { + Name: "netudpgroup.0", + In: `package main + +import "net" + +func f() { + err := x.JoinGroup(gaddr) + err = y.LeaveGroup(gaddr) +} +`, + Out: `package main + +import "net" + +func f() { + err := x.JoinGroup(nil, gaddr) + err = y.LeaveGroup(nil, gaddr) +} +`, + }, +} diff --git a/src/cmd/gofix/osopen.go b/src/cmd/gofix/osopen.go index 56147c390..19c19b5b6 100644 --- a/src/cmd/gofix/osopen.go +++ b/src/cmd/gofix/osopen.go @@ -31,6 +31,7 @@ func osopen(f *ast.File) bool { // Rename O_CREAT to O_CREATE. if expr, ok := n.(ast.Expr); ok && isPkgDot(expr, "os", "O_CREAT") { expr.(*ast.SelectorExpr).Sel.Name = "O_CREATE" + fixed = true return } diff --git a/src/cmd/gofix/osopen_test.go b/src/cmd/gofix/osopen_test.go index 43ddd1a40..a33bcd4fb 100644 --- a/src/cmd/gofix/osopen_test.go +++ b/src/cmd/gofix/osopen_test.go @@ -54,6 +54,29 @@ func f() { os.OpenFile(a, os.O_SURPRISE|os.O_CREATE, 0666) _ = os.O_CREATE } +`, + }, + { + Name: "osopen.1", + In: `package main + +import ( + "os" +) + +func f() { + _ = os.O_CREAT +} +`, + Out: `package main + +import ( + "os" +) + +func f() { + _ = os.O_CREATE +} `, }, } diff --git a/src/cmd/gofix/url.go b/src/cmd/gofix/url.go index c1e47bd4e..7135d8edf 100644 --- a/src/cmd/gofix/url.go +++ b/src/cmd/gofix/url.go @@ -27,6 +27,7 @@ func init() { } var urlRenames = []struct{ in, out string }{ + {"URL", "URL"}, {"ParseURL", "Parse"}, {"ParseURLReference", "ParseWithReference"}, {"ParseQuery", "ParseQuery"}, @@ -45,7 +46,12 @@ func url(f *ast.File) bool { fixed := false // Update URL code. + var skip interface{} urlWalk := func(n interface{}) { + if n == skip { + skip = nil + return + } // Is it an identifier? if ident, ok := n.(*ast.Ident); ok && ident.Name == "url" { ident.Name = "url_" @@ -56,6 +62,12 @@ func url(f *ast.File) bool { fixed = urlDoFields(fn.Params) || fixed fixed = urlDoFields(fn.Results) || fixed } + // U{url: ...} is likely a struct field. + if kv, ok := n.(*ast.KeyValueExpr); ok { + if ident, ok := kv.Key.(*ast.Ident); ok && ident.Name == "url" { + skip = ident + } + } } // Fix up URL code and add import, at most once. @@ -63,7 +75,7 @@ func url(f *ast.File) bool { if fixed { return } - walk(f, urlWalk) + walkBeforeAfter(f, urlWalk, nop) addImport(f, "url") fixed = true } diff --git a/src/cmd/gofix/url_test.go b/src/cmd/gofix/url_test.go index 1a7095a5d..8d9542cbc 100644 --- a/src/cmd/gofix/url_test.go +++ b/src/cmd/gofix/url_test.go @@ -18,6 +18,7 @@ import ( ) func f() { + var _ http.URL http.ParseURL(a) http.ParseURLReference(a) http.ParseQuery(a) @@ -33,6 +34,7 @@ func f() { import "url" func f() { + var _ url.URL url.Parse(a) url.ParseWithReference(a) url.ParseQuery(a) @@ -78,10 +80,15 @@ import ( "http" ) +type U struct{ url int } +type M map[int]int + func f() { http.ParseURL(a) var url = 23 url, x := 45, y + _ = U{url: url} + _ = M{url + 1: url} } func g(url string) string { @@ -96,10 +103,15 @@ func h() (url string) { import "url" +type U struct{ url int } +type M map[int]int + func f() { url.Parse(a) var url_ = 23 url_, x := 45, y + _ = U{url: url_} + _ = M{url_ + 1: url_} } func g(url_ string) string { diff --git a/src/cmd/goinstall/Makefile b/src/cmd/goinstall/Makefile index f61354f39..b90646973 100644 --- a/src/cmd/goinstall/Makefile +++ b/src/cmd/goinstall/Makefile @@ -11,3 +11,9 @@ GOFILES=\ make.go\ include ../../Make.cmd + +test: + gotest + +testshort: + gotest -test.short diff --git a/src/cmd/goinstall/doc.go b/src/cmd/goinstall/doc.go index 8260cb4d7..47c615364 100644 --- a/src/cmd/goinstall/doc.go +++ b/src/cmd/goinstall/doc.go @@ -94,8 +94,11 @@ attempt to fetch updates. The -u flag changes this behavior, causing goinstall to update all remote packages encountered during the installation. -When downloading or updating, goinstall first looks for a tag or branch -named "release". If there is one, it uses that version of the code. +When downloading or updating, goinstall looks for a tag with the "go." prefix +that corresponds to the local Go version. For Go "release.r58" it looks for a +tag named "go.r58". For "weekly.2011-06-03" it looks for "go.weekly.2011-06-03". +If the specific "go.X" tag is not found, it chooses the closest earlier version. +If an appropriate tag is found, goinstall uses that version of the code. Otherwise it uses the default version selected by the version control system, typically HEAD for git, tip for Mercurial. diff --git a/src/cmd/goinstall/download.go b/src/cmd/goinstall/download.go index 3e9927c3d..cc873150a 100644 --- a/src/cmd/goinstall/download.go +++ b/src/cmd/goinstall/download.go @@ -7,12 +7,15 @@ package main import ( + "bytes" "exec" "fmt" "http" "os" "path/filepath" "regexp" + "runtime" + "strconv" "strings" ) @@ -36,22 +39,21 @@ func maybeReportToDashboard(path string) { // a vcs represents a version control system // like Mercurial, Git, or Subversion. type vcs struct { - name string - cmd string - metadir string - checkout string - clone string - update string - updateReleaseFlag string - pull string - pullForceFlag string - log string - logLimitFlag string - logReleaseFlag string - check string - protocols []string - suffix string - defaultHosts []host + name string + cmd string + metadir string + checkout string + clone string + update string + updateRevFlag string + pull string + pullForceFlag string + tagList string + tagListRe *regexp.Regexp + check string + protocols []string + suffix string + defaultHosts []host } type host struct { @@ -61,20 +63,18 @@ type host struct { } var hg = vcs{ - name: "Mercurial", - cmd: "hg", - metadir: ".hg", - checkout: "checkout", - clone: "clone", - update: "update", - updateReleaseFlag: "release", - pull: "pull", - log: "log", - logLimitFlag: "-l1", - logReleaseFlag: "-rrelease", - check: "identify", - protocols: []string{"https", "http"}, - suffix: ".hg", + name: "Mercurial", + cmd: "hg", + metadir: ".hg", + checkout: "checkout", + clone: "clone", + update: "update", + pull: "pull", + tagList: "tags", + tagListRe: regexp.MustCompile("([^ ]+)[^\n]+\n"), + check: "identify", + protocols: []string{"https", "http"}, + suffix: ".hg", defaultHosts: []host{ {regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/hg)(/[a-z0-9A-Z_.\-/]*)?$`), "https", ""}, {regexp.MustCompile(`^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`), "http", ""}, @@ -82,20 +82,18 @@ var hg = vcs{ } var git = vcs{ - name: "Git", - cmd: "git", - metadir: ".git", - checkout: "checkout", - clone: "clone", - update: "pull", - updateReleaseFlag: "release", - pull: "fetch", - log: "show-ref", - logLimitFlag: "", - logReleaseFlag: "release", - check: "ls-remote", - protocols: []string{"git", "https", "http"}, - suffix: ".git", + name: "Git", + cmd: "git", + metadir: ".git", + checkout: "checkout", + clone: "clone", + update: "pull", + pull: "fetch", + tagList: "tag", + tagListRe: regexp.MustCompile("([^\n]+)\n"), + check: "ls-remote", + protocols: []string{"git", "https", "http"}, + suffix: ".git", defaultHosts: []host{ {regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/git)(/[a-z0-9A-Z_.\-/]*)?$`), "https", ""}, {regexp.MustCompile(`^(github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]*)?$`), "http", ".git"}, @@ -103,40 +101,35 @@ var git = vcs{ } var svn = vcs{ - name: "Subversion", - cmd: "svn", - metadir: ".svn", - checkout: "checkout", - clone: "checkout", - update: "update", - updateReleaseFlag: "release", - log: "log", - logLimitFlag: "-l1", - logReleaseFlag: "release", - check: "info", - protocols: []string{"https", "http", "svn"}, - suffix: ".svn", + name: "Subversion", + cmd: "svn", + metadir: ".svn", + checkout: "checkout", + clone: "checkout", + update: "update", + check: "info", + protocols: []string{"https", "http", "svn"}, + suffix: ".svn", defaultHosts: []host{ {regexp.MustCompile(`^([a-z0-9\-]+\.googlecode\.com/svn)(/[a-z0-9A-Z_.\-/]*)?$`), "https", ""}, }, } var bzr = vcs{ - name: "Bazaar", - cmd: "bzr", - metadir: ".bzr", - checkout: "update", - clone: "branch", - update: "update", - updateReleaseFlag: "-rrelease", - pull: "pull", - pullForceFlag: "--overwrite", - log: "log", - logLimitFlag: "-l1", - logReleaseFlag: "-rrelease", - check: "info", - protocols: []string{"https", "http", "bzr"}, - suffix: ".bzr", + name: "Bazaar", + cmd: "bzr", + metadir: ".bzr", + checkout: "update", + clone: "branch", + update: "update", + updateRevFlag: "-r", + pull: "pull", + pullForceFlag: "--overwrite", + tagList: "tags", + tagListRe: regexp.MustCompile("([^ ]+)[^\n]+\n"), + check: "info", + protocols: []string{"https", "http", "bzr"}, + suffix: ".bzr", defaultHosts: []host{ {regexp.MustCompile(`^(launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+))(/[a-z0-9A-Z_.\-/]+)?$`), "https", ""}, }, @@ -240,20 +233,84 @@ func download(pkg, srcDir string) (public bool, err os.Error) { return } -// Try to detect if a "release" tag exists. If it does, update -// to the tagged version, otherwise just update the current branch. -// NOTE(_nil): svn will always fail because it is trying to get -// the revision history of a file named "release" instead of -// looking for a commit with a release tag +// updateRepo gets a list of tags in the repository and +// checks out the tag closest to the current runtime.Version. +// If no matching tag is found, it just updates to tip. func (v *vcs) updateRepo(dst string) os.Error { - if err := quietRun(dst, nil, v.cmd, v.log, v.logLimitFlag, v.logReleaseFlag); err == nil { - if err := run(dst, nil, v.cmd, v.checkout, v.updateReleaseFlag); err != nil { - return err - } - } else if err := run(dst, nil, v.cmd, v.update); err != nil { + if v.tagList == "" || v.tagListRe == nil { + // TODO(adg): fix for svn + return run(dst, nil, v.cmd, v.update) + } + + // Get tag list. + stderr := new(bytes.Buffer) + cmd := exec.Command(v.cmd, v.tagList) + cmd.Dir = dst + cmd.Stderr = stderr + b, err := cmd.Output() + if err != nil { + errorf("%s %s: %s\n", v.cmd, v.tagList, stderr) return err } - return nil + var tags []string + for _, m := range v.tagListRe.FindAllStringSubmatch(string(b), -1) { + tags = append(tags, m[1]) + } + + // Only use the tag component of runtime.Version. + ver := strings.Split(runtime.Version(), " ")[0] + + // Select tag. + if tag := selectTag(ver, tags); tag != "" { + printf("selecting revision %q\n", tag) + return run(dst, nil, v.cmd, v.checkout, v.updateRevFlag+tag) + } + + // No matching tag found, make default selection. + printf("selecting tip\n") + return run(dst, nil, v.cmd, v.update) +} + +// selectTag returns the closest matching tag for a given version. +// Closest means the latest one that is not after the current release. +// Version "release.rN" matches tags of the form "go.rN" (N being a decimal). +// Version "weekly.YYYY-MM-DD" matches tags like "go.weekly.YYYY-MM-DD". +func selectTag(goVersion string, tags []string) (match string) { + const rPrefix = "release.r" + if strings.HasPrefix(goVersion, rPrefix) { + p := "go.r" + v, err := strconv.Atof64(goVersion[len(rPrefix):]) + if err != nil { + return "" + } + var matchf float64 + for _, t := range tags { + if !strings.HasPrefix(t, p) { + continue + } + tf, err := strconv.Atof64(t[len(p):]) + if err != nil { + continue + } + if matchf < tf && tf <= v { + match, matchf = t, tf + } + } + } + const wPrefix = "weekly." + if strings.HasPrefix(goVersion, wPrefix) { + p := "go.weekly." + v := goVersion[len(wPrefix):] + for _, t := range tags { + if !strings.HasPrefix(t, p) { + continue + } + if match < t && t[len(p):] <= v { + match = t + } + } + } + return match } // checkoutRepo checks out repo into dst using vcs. diff --git a/src/cmd/goinstall/main.go b/src/cmd/goinstall/main.go index 910ab7090..23b26e383 100644 --- a/src/cmd/goinstall/main.go +++ b/src/cmd/goinstall/main.go @@ -20,8 +20,8 @@ import ( ) func usage() { - fmt.Fprint(os.Stderr, "usage: goinstall importpath...\n") - fmt.Fprintf(os.Stderr, "\tgoinstall -a\n") + fmt.Fprintln(os.Stderr, "usage: goinstall [flags] importpath...") + fmt.Fprintln(os.Stderr, " goinstall [flags] -a") flag.PrintDefaults() os.Exit(2) } @@ -71,6 +71,13 @@ func errorf(format string, args ...interface{}) { logf(format, args...) } +func terrorf(tree *build.Tree, format string, args ...interface{}) { + if tree != nil && tree.Goroot && os.Getenv("GOPATH") == "" { + format = strings.TrimRight(format, "\n") + " ($GOPATH not set)\n" + } + errorf(format, args...) +} + func main() { flag.Usage = usage flag.Parse() @@ -156,7 +163,7 @@ func logPackage(pkg string, tree *build.Tree) (logged bool) { name := filepath.Join(tree.Path, logfile) fout, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) if err != nil { - logf("%s\n", err) + terrorf(tree, "package log: %s\n", err) return false } fmt.Fprintf(fout, "%s\n", pkg) @@ -182,6 +189,12 @@ func install(pkg, parent string) { visit[pkg] = done }() + // Don't allow trailing '/' + if _, f := filepath.Split(pkg); f == "" { + errorf("%s should not have trailing '/'\n", pkg) + return + } + // Check whether package is local or remote. // If remote, download or update it. tree, pkg, err := build.FindTree(pkg) @@ -209,7 +222,7 @@ func install(pkg, parent string) { } } if err != nil { - errorf("%s: %v\n", pkg, err) + terrorf(tree, "%s: %v\n", pkg, err) return } dir := filepath.Join(tree.SrcDir(), pkg) @@ -217,11 +230,11 @@ func install(pkg, parent string) { // Install prerequisites. dirInfo, err := build.ScanDir(dir, parent == "") if err != nil { - errorf("%s: %v\n", pkg, err) + terrorf(tree, "%s: %v\n", pkg, err) return } if len(dirInfo.GoFiles)+len(dirInfo.CgoFiles) == 0 { - errorf("%s: package has no files\n", pkg) + terrorf(tree, "%s: package has no files\n", pkg) return } for _, p := range dirInfo.Imports { @@ -237,13 +250,13 @@ func install(pkg, parent string) { if *useMake { err := domake(dir, pkg, tree, dirInfo.IsCommand()) if err != nil { - errorf("%s: install: %v\n", pkg, err) + terrorf(tree, "%s: install: %v\n", pkg, err) return } } else { script, err := build.Build(tree, pkg, dirInfo) if err != nil { - errorf("%s: install: %v\n", pkg, err) + terrorf(tree, "%s: install: %v\n", pkg, err) return } if *nuke { @@ -257,7 +270,7 @@ func install(pkg, parent string) { if script.Stale() { printf("%s: install\n", pkg) if err := script.Run(); err != nil { - errorf("%s: install: %v\n", pkg, err) + terrorf(tree, "%s: install: %v\n", pkg, err) return } } else { diff --git a/src/cmd/goinstall/tag_test.go b/src/cmd/goinstall/tag_test.go new file mode 100644 index 000000000..a23a7ea82 --- /dev/null +++ b/src/cmd/goinstall/tag_test.go @@ -0,0 +1,73 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "testing" + +var selectTagTestTags = []string{ + "go.r58", + "go.r58.1", + "go.r59", + "go.r59.1", + "go.r61", + "go.r61.1", + "go.weekly.2010-01-02", + "go.weekly.2011-10-12", + "go.weekly.2011-10-12.1", + "go.weekly.2011-10-14", + "go.weekly.2011-11-01", + // these should be ignored: + "release.r59", + "release.r59.1", + "release", + "weekly.2011-10-12", + "weekly.2011-10-12.1", + "weekly", + "foo", + "bar", + "go.f00", + "go!r60", + "go.1999-01-01", +} + +var selectTagTests = []struct { + version string + selected string +}{ + {"release.r57", ""}, + {"release.r58.2", "go.r58.1"}, + {"release.r59", "go.r59"}, + {"release.r59.1", "go.r59.1"}, + {"release.r60", "go.r59.1"}, + {"release.r60.1", "go.r59.1"}, + {"release.r61", "go.r61"}, + {"release.r66", "go.r61.1"}, + {"weekly.2010-01-01", ""}, + {"weekly.2010-01-02", "go.weekly.2010-01-02"}, + {"weekly.2010-01-02.1", "go.weekly.2010-01-02"}, + {"weekly.2010-01-03", "go.weekly.2010-01-02"}, + {"weekly.2011-10-12", "go.weekly.2011-10-12"}, + {"weekly.2011-10-12.1", "go.weekly.2011-10-12.1"}, + {"weekly.2011-10-13", "go.weekly.2011-10-12.1"}, + {"weekly.2011-10-14", "go.weekly.2011-10-14"}, + {"weekly.2011-10-14.1", "go.weekly.2011-10-14"}, + {"weekly.2011-11-01", "go.weekly.2011-11-01"}, + {"weekly.2014-01-01", "go.weekly.2011-11-01"}, + {"weekly.3000-01-01", "go.weekly.2011-11-01"}, + // faulty versions: + {"release.f00", ""}, + {"weekly.1999-01-01", ""}, + {"junk", ""}, + {"", ""}, +} + +func TestSelectTag(t *testing.T) { + for _, c := range selectTagTests { + selected := selectTag(c.version, selectTagTestTags) + if selected != c.selected { + t.Errorf("selectTag(%q) = %q, want %q", c.version, selected, c.selected) + } + } +} diff --git a/src/cmd/gopack/ar.c b/src/cmd/gopack/ar.c index 0b5e608c7..96f36605f 100644 --- a/src/cmd/gopack/ar.c +++ b/src/cmd/gopack/ar.c @@ -1532,6 +1532,8 @@ arwrite(int fd, Armember *bp) int page(Arfile *ap) { + USED(ap); + sysfatal("page"); return 1; } diff --git a/src/cmd/gotest/gotest.go b/src/cmd/gotest/gotest.go index 4cb3da23c..8e3a42232 100644 --- a/src/cmd/gotest/gotest.go +++ b/src/cmd/gotest/gotest.go @@ -9,12 +9,13 @@ import ( "exec" "fmt" "go/ast" + "go/build" "go/parser" "go/token" "io/ioutil" "os" - "path/filepath" "runtime" + "sort" "strings" "time" "unicode" @@ -159,17 +160,19 @@ func setEnvironment() { } // getTestFileNames gets the set of files we're looking at. -// If gotest has no arguments, it scans for file names matching "[^.]*_test.go". +// If gotest has no arguments, it scans the current directory +// for test files. func getTestFileNames() { names := fileNames if len(names) == 0 { - var err os.Error - names, err = filepath.Glob("[^.]*_test.go") + info, err := build.ScanDir(".", true) if err != nil { - Fatalf("Glob pattern error: %s", err) + Fatalf("scanning directory: %v", err) } + names = append(info.TestGoFiles, info.XTestGoFiles...) + sort.Strings(names) if len(names) == 0 { - Fatalf(`no test files found: no match for "[^.]*_test.go"`) + Fatalf("no test files found in current directory") } } for _, n := range names { diff --git a/src/cmd/gotry/gotry b/src/cmd/gotry/gotry index 3cc7a9864..c81b6c7d0 100755 --- a/src/cmd/gotry/gotry +++ b/src/cmd/gotry/gotry @@ -112,7 +112,7 @@ functions=$(getFunctions) # Write file to compile file="/tmp/$USER.try" -rm -f "file.go" +rm -f "$file.go" ( cat <<'!' package main diff --git a/src/cmd/gotype/testdata/test1.go b/src/cmd/gotype/testdata/test1.go index 0bd46568d..a3298e6e5 100644 --- a/src/cmd/gotype/testdata/test1.go +++ b/src/cmd/gotype/testdata/test1.go @@ -4,3 +4,20 @@ func _() { // the scope of a local type declaration starts immediately after the type name type T struct{ _ *T } } + +func _(x interface{}) { + // the variable defined by a TypeSwitchGuard is declared in each TypeCaseClause + switch t := x.(type) { + case int: + _ = t + case float32: + _ = t + default: + _ = t + } + + // the variable defined by a TypeSwitchGuard must not conflict with other + // variables declared in the initial simple statement + switch t := 0; t := x.(type) { + } +} diff --git a/src/cmd/hgpatch/main.go b/src/cmd/hgpatch/main.go index 9e338abcb..d4169ae85 100644 --- a/src/cmd/hgpatch/main.go +++ b/src/cmd/hgpatch/main.go @@ -6,7 +6,6 @@ package main import ( "bytes" - "container/vector" "exec" "flag" "fmt" @@ -242,15 +241,17 @@ func chk(err os.Error) { // Undo log type undo func() os.Error -var undoLog vector.Vector // vector of undo +var undoLog []undo -func undoRevert(name string) { undoLog.Push(undo(func() os.Error { return hgRevert(name) })) } +func undoRevert(name string) { + undoLog = append(undoLog, undo(func() os.Error { return hgRevert(name) })) +} -func undoRm(name string) { undoLog.Push(undo(func() os.Error { return os.Remove(name) })) } +func undoRm(name string) { undoLog = append(undoLog, undo(func() os.Error { return os.Remove(name) })) } func runUndo() { - for i := undoLog.Len() - 1; i >= 0; i-- { - if err := undoLog.At(i).(undo)(); err != nil { + for i := len(undoLog) - 1; i >= 0; i-- { + if err := undoLog[i](); err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) } } diff --git a/src/cmd/ld/data.c b/src/cmd/ld/data.c index e7269169e..a7f61c927 100644 --- a/src/cmd/ld/data.c +++ b/src/cmd/ld/data.c @@ -182,7 +182,11 @@ relocsym(Sym *s) o = symaddr(r->sym) + r->add; break; case D_PCREL: - o = symaddr(r->sym) + r->add - (s->value + r->off + r->siz); + // r->sym can be null when CALL $(constant) is transformed from absoulte PC to relative PC call. + o = 0; + if(r->sym) + o += symaddr(r->sym); + o += r->add - (s->value + r->off + r->siz); break; case D_SIZE: o = r->sym->size + r->add; diff --git a/src/cmd/ld/elf.c b/src/cmd/ld/elf.c index f9f9ef6b2..00cfc8c8c 100644 --- a/src/cmd/ld/elf.c +++ b/src/cmd/ld/elf.c @@ -77,6 +77,19 @@ elf64phdr(ElfPhdr *e) void elf32phdr(ElfPhdr *e) { + int frag; + + if(e->type == PT_LOAD) { + // Correct ELF loaders will do this implicitly, + // but buggy ELF loaders like the one in some + // versions of QEMU won't. + frag = e->vaddr&(e->align-1); + e->off -= frag; + e->vaddr -= frag; + e->paddr -= frag; + e->filesz += frag; + e->memsz += frag; + } LPUT(e->type); LPUT(e->off); LPUT(e->vaddr); diff --git a/src/cmd/ld/lib.c b/src/cmd/ld/lib.c index 5d1e6d61b..37379e186 100644 --- a/src/cmd/ld/lib.c +++ b/src/cmd/ld/lib.c @@ -295,19 +295,33 @@ nextar(Biobuf *bp, int off, struct ar_hdr *a) { int r; int32 arsize; + char *buf; if (off&01) off++; Bseek(bp, off, 0); - r = Bread(bp, a, SAR_HDR); - if(r != SAR_HDR) - return 0; - if(strncmp(a->fmag, ARFMAG, sizeof(a->fmag))) + buf = Brdline(bp, '\n'); + r = Blinelen(bp); + if(buf == nil) { + if(r == 0) + return 0; + return -1; + } + if(r == SAR_HDR) { + memmove(a, buf, SAR_HDR); + } else if (r == SAR_HDR-SARNAME+16) { // old Plan 9 + memset(a->name, ' ', sizeof a->name); + memmove(a, buf, 16); + memmove((char*)a+SARNAME, buf+16, SAR_HDR-SARNAME); + } else { // unexpected + return -1; + } + if(strncmp(a->fmag, ARFMAG, sizeof a->fmag)) return -1; arsize = strtol(a->size, 0, 0); if (arsize&1) arsize++; - return arsize + SAR_HDR; + return arsize + r; } void @@ -1332,7 +1346,7 @@ Yconv(Fmt *fp) fmtprint(fp, ""); } else { fmtstrinit(&fmt); - fmtprint(&fmt, "%s @0x%08x [%d]", s->name, s->value, s->size); + fmtprint(&fmt, "%s @0x%08llx [%lld]", s->name, (vlong)s->value, (vlong)s->size); for (i = 0; i < s->size; i++) { if (!(i%8)) fmtprint(&fmt, "\n\t0x%04x ", i); fmtprint(&fmt, "%02x ", s->p[i]); diff --git a/src/cmd/ld/pe.c b/src/cmd/ld/pe.c index 334c9959f..df6c95976 100644 --- a/src/cmd/ld/pe.c +++ b/src/cmd/ld/pe.c @@ -32,6 +32,11 @@ static char dosstub[] = 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; +// Note: currently only up to 8 chars plus \0. +static char *symlabels[] = { + "symtab", "esymtab", "pclntab", "epclntab" +}; + static Sym *rsrcsym; static char symnames[256]; @@ -44,6 +49,7 @@ static int pe64; static int nsect; static int nextsectoff; static int nextfileoff; +static int textsect; static IMAGE_FILE_HEADER fh; static IMAGE_OPTIONAL_HEADER oh; @@ -449,20 +455,29 @@ addsymtable(void) { IMAGE_SECTION_HEADER *h; int i, size; + Sym *s; - if(nextsymoff == 0) - return; - - size = nextsymoff + 4 + 18; + fh.NumberOfSymbols = sizeof(symlabels)/sizeof(symlabels[0]); + size = nextsymoff + 4 + 18*fh.NumberOfSymbols; h = addpesection(".symtab", size, size); h->Characteristics = IMAGE_SCN_MEM_READ| IMAGE_SCN_MEM_DISCARDABLE; chksectoff(h, cpos()); fh.PointerToSymbolTable = cpos(); - fh.NumberOfSymbols = 1; - strnput("", 18); // one empty symbol - // put symbol string table - lputl(size); + + // put COFF symbol table + for (i=0; iname, 8); + lputl(datoff(s->value)); + wputl(textsect); + wputl(0x0308); // "array of structs" + cput(2); // storage class: external + cput(0); // no aux entries + } + + // put COFF string table + lputl(nextsymoff + 4); for (i=0; iSizeOfRawData - size); @@ -510,6 +525,48 @@ addpersrc(void) dd[IMAGE_DIRECTORY_ENTRY_RESOURCE].Size = h->VirtualSize; } +static void +addexcept(IMAGE_SECTION_HEADER *text) +{ + IMAGE_SECTION_HEADER *pdata, *xdata; + vlong startoff; + uvlong n; + Sym *sym; + + if(thechar != '6') + return; + + // write unwind info + sym = lookup("runtime.sigtramp", 0); + startoff = cpos(); + lputl(9); // version=1, flags=UNW_FLAG_EHANDLER, rest 0 + lputl(sym->value - PEBASE); + lputl(0); + + n = cpos() - startoff; + xdata = addpesection(".xdata", n, n); + xdata->Characteristics = IMAGE_SCN_MEM_READ| + IMAGE_SCN_CNT_INITIALIZED_DATA; + chksectoff(xdata, startoff); + strnput("", xdata->SizeOfRawData - n); + + // write a function table entry for the whole text segment + startoff = cpos(); + lputl(text->VirtualAddress); + lputl(text->VirtualAddress + text->VirtualSize); + lputl(xdata->VirtualAddress); + + n = cpos() - startoff; + pdata = addpesection(".pdata", n, n); + pdata->Characteristics = IMAGE_SCN_MEM_READ| + IMAGE_SCN_CNT_INITIALIZED_DATA; + chksectoff(pdata, startoff); + strnput("", pdata->SizeOfRawData - n); + + dd[IMAGE_DIRECTORY_ENTRY_EXCEPTION].VirtualAddress = pdata->VirtualAddress; + dd[IMAGE_DIRECTORY_ENTRY_EXCEPTION].Size = pdata->VirtualSize; +} + void asmbpe(void) { @@ -532,6 +589,7 @@ asmbpe(void) IMAGE_SCN_CNT_INITIALIZED_DATA| IMAGE_SCN_MEM_EXECUTE|IMAGE_SCN_MEM_READ; chksectseg(t, &segtext); + textsect = nsect; d = addpesection(".data", segdata.len, segdata.filelen); d->Characteristics = IMAGE_SCN_CNT_INITIALIZED_DATA| @@ -546,7 +604,8 @@ asmbpe(void) addexports(); addsymtable(); addpersrc(); - + addexcept(t); + fh.NumberOfSections = nsect; fh.TimeDateStamp = time(0); fh.Characteristics = IMAGE_FILE_RELOCS_STRIPPED| @@ -583,8 +642,16 @@ asmbpe(void) set(Subsystem, IMAGE_SUBSYSTEM_WINDOWS_GUI); else set(Subsystem, IMAGE_SUBSYSTEM_WINDOWS_CUI); - set(SizeOfStackReserve, 0x0040000); - set(SizeOfStackCommit, 0x00001000); + + // Disable stack growth as we don't want Windows to + // fiddle with the thread stack limits, which we set + // ourselves to circumvent the stack checks in the + // Windows exception dispatcher. + // Commit size must be strictly less than reserve + // size otherwise reserve will be rounded up to a + // larger size, as verified with VMMap. + set(SizeOfStackReserve, 0x00010000); + set(SizeOfStackCommit, 0x0000ffff); set(SizeOfHeapReserve, 0x00100000); set(SizeOfHeapCommit, 0x00001000); set(NumberOfRvaAndSizes, 16); diff --git a/src/cmd/ld/symtab.c b/src/cmd/ld/symtab.c index 60e146b35..00413af00 100644 --- a/src/cmd/ld/symtab.c +++ b/src/cmd/ld/symtab.c @@ -34,10 +34,7 @@ #include "../ld/lib.h" #include "../ld/elf.h" -char *elfstrdat; -int elfstrsize; -int maxelfstr; -int elftextsh; +static int maxelfstr; int putelfstr(char *s) diff --git a/src/cmd/prof/Makefile b/src/cmd/prof/Makefile index 8a1a2f308..6cefceb8e 100644 --- a/src/cmd/prof/Makefile +++ b/src/cmd/prof/Makefile @@ -25,6 +25,7 @@ endif install: install-$(NAME) install-pprof install-linux: install-default install-freebsd: install-default +install-openbsd: install-default install-windows: install-default # on Darwin, have to install and setgid; see $GOROOT/src/sudo.bash diff --git a/src/cmd/prof/gopprof b/src/cmd/prof/gopprof index be5f84e9e..83438b7cd 100755 --- a/src/cmd/prof/gopprof +++ b/src/cmd/prof/gopprof @@ -1241,7 +1241,7 @@ sub Disassemble { while () { s/\r//g; # turn windows-looking lines into unix-looking lines chop; - if (m|\s*([^:\s]+):(\d+)\s*$|) { + if (m|\s*(.+):(\d+)\s*$|) { # Location line of the form: # : $filename = $1; @@ -4485,7 +4485,7 @@ sub MapSymbolsWithNM { sub ShortFunctionName { my $function = shift; - while ($function =~ s/\([^()]*\)(\s*const)?//g) { } # Argument types + while ($function =~ s/(?]*>//g) { } # Remove template arguments $function =~ s/^.*\s+(\w+::)/$1/; # Remove leading type return $function; @@ -4494,7 +4494,7 @@ sub ShortFunctionName { # Trim overly long symbols found in disassembler output sub CleanDisassembly { my $d = shift; - while ($d =~ s/\([^()%]*\)(\s*const)?//g) { } # Argument types, not (%rax) + while ($d =~ s/(?]*>/$1/g) { } # Remove template arguments return $d; } diff --git a/src/cmd/prof/main.c b/src/cmd/prof/main.c index f36759cd3..f0acaf1c0 100644 --- a/src/cmd/prof/main.c +++ b/src/cmd/prof/main.c @@ -386,6 +386,8 @@ addtohistogram(uvlong pc, uvlong callerpc, uvlong sp) { int h; PC *x; + + USED(sp); h = (pc + callerpc*101) % Ncounters; for(x = counters[h]; x != NULL; x = x->next) { @@ -437,6 +439,8 @@ uvlong nextpc; void xptrace(Map *map, uvlong pc, uvlong sp, Symbol *sym) { + USED(map); + char buf[1024]; if(sym == nil){ fprint(2, "syms\n"); diff --git a/src/lib9/exitcode.c b/src/lib9/exitcode.c index 234492acf..a952b2da2 100644 --- a/src/lib9/exitcode.c +++ b/src/lib9/exitcode.c @@ -29,6 +29,7 @@ THE SOFTWARE. int exitcode(char *s) { + USED(s); return 1; } diff --git a/src/libmach/executable.c b/src/libmach/executable.c index e90334438..9d4532302 100644 --- a/src/libmach/executable.c +++ b/src/libmach/executable.c @@ -66,6 +66,7 @@ static int adotout(int, Fhdr*, ExecHdr*); static int elfdotout(int, Fhdr*, ExecHdr*); static int machdotout(int, Fhdr*, ExecHdr*); static int armdotout(int, Fhdr*, ExecHdr*); +static int pedotout(int, Fhdr*, ExecHdr*); static void setsym(Fhdr*, vlong, int32, vlong, int32, vlong, int32); static void setdata(Fhdr*, uvlong, int32, vlong, int32); static void settext(Fhdr*, uvlong, uvlong, int32, vlong); @@ -312,6 +313,15 @@ ExecTable exectab[] = sizeof(Exec), beswal, common }, + { 0x4d5a9000, /* see dosstub[] in pe.c */ + "windows PE executable", + nil, + FWINPE, + 0, + &mi386, + sizeof(Exec), /* TODO */ + nil, + pedotout }, { 0 }, }; @@ -502,6 +512,8 @@ commonllp64(int unused, Fhdr *fp, ExecHdr *hp) int32 pgsize; uvlong entry; + USED(unused); + hswal(&hp->e, sizeof(Exec)/sizeof(int32), beswal); if(!(hp->e.exechdr.magic & HDR_MAGIC)) return 0; @@ -542,6 +554,10 @@ commonllp64(int unused, Fhdr *fp, ExecHdr *hp) static int mipsboot(int fd, Fhdr *fp, ExecHdr *hp) { + USED(fd); + USED(fp); + USED(hp); + abort(); #ifdef unused USED(fd); @@ -573,6 +589,10 @@ abort(); static int mips4kboot(int fd, Fhdr *fp, ExecHdr *hp) { + USED(fd); + USED(fp); + USED(hp); + abort(); #ifdef unused USED(fd); @@ -604,6 +624,10 @@ abort(); static int sparcboot(int fd, Fhdr *fp, ExecHdr *hp) { + USED(fd); + USED(fp); + USED(hp); + abort(); #ifdef unused USED(fd); @@ -624,6 +648,10 @@ abort(); static int nextboot(int fd, Fhdr *fp, ExecHdr *hp) { + USED(fd); + USED(fp); + USED(hp); + abort(); #ifdef unused USED(fd); @@ -645,7 +673,6 @@ abort(); static int elf64dotout(int fd, Fhdr *fp, ExecHdr *hp) { - uvlong (*swav)(uvlong); uint32 (*swal)(uint32); ushort (*swab)(ushort); @@ -784,7 +811,7 @@ elf64dotout(int fd, Fhdr *fp, ExecHdr *hp) buf = malloc(sh[ep->shstrndx].size); if (buf == 0) goto done; - memset(buf, 0, sizeof buf); + memset(buf, 0, sh[ep->shstrndx].size); seek(fd, sh[ep->shstrndx].offset, 0); i = read(fd, buf, sh[ep->shstrndx].size); USED(i); // shut up ubuntu gcc @@ -962,7 +989,7 @@ elfdotout(int fd, Fhdr *fp, ExecHdr *hp) buf = malloc(sh[ep->shstrndx].size); if (buf == 0) goto done; - memset(buf, 0, sizeof buf); + memset(buf, 0, sh[ep->shstrndx].size); seek(fd, sh[ep->shstrndx].offset, 0); i = read(fd, buf, sh[ep->shstrndx].size); USED(i); // shut up ubuntu gcc @@ -1232,6 +1259,164 @@ armdotout(int fd, Fhdr *fp, ExecHdr *hp) return 1; } +/* + * Structures needed to parse PE image. + */ +typedef struct { + uint16 Machine; + uint16 NumberOfSections; + uint32 TimeDateStamp; + uint32 PointerToSymbolTable; + uint32 NumberOfSymbols; + uint16 SizeOfOptionalHeader; + uint16 Characteristics; +} IMAGE_FILE_HEADER; + +typedef struct { + uint8 Name[8]; + uint32 VirtualSize; + uint32 VirtualAddress; + uint32 SizeOfRawData; + uint32 PointerToRawData; + uint32 PointerToRelocations; + uint32 PointerToLineNumbers; + uint16 NumberOfRelocations; + uint16 NumberOfLineNumbers; + uint32 Characteristics; +} IMAGE_SECTION_HEADER; + +typedef struct { + uint32 VirtualAddress; + uint32 Size; +} IMAGE_DATA_DIRECTORY; + +typedef struct { + uint16 Magic; + uint8 MajorLinkerVersion; + uint8 MinorLinkerVersion; + uint32 SizeOfCode; + uint32 SizeOfInitializedData; + uint32 SizeOfUninitializedData; + uint32 AddressOfEntryPoint; + uint32 BaseOfCode; + uint32 BaseOfData; + uint32 ImageBase; + uint32 SectionAlignment; + uint32 FileAlignment; + uint16 MajorOperatingSystemVersion; + uint16 MinorOperatingSystemVersion; + uint16 MajorImageVersion; + uint16 MinorImageVersion; + uint16 MajorSubsystemVersion; + uint16 MinorSubsystemVersion; + uint32 Win32VersionValue; + uint32 SizeOfImage; + uint32 SizeOfHeaders; + uint32 CheckSum; + uint16 Subsystem; + uint16 DllCharacteristics; + uint32 SizeOfStackReserve; + uint32 SizeOfStackCommit; + uint32 SizeOfHeapReserve; + uint32 SizeOfHeapCommit; + uint32 LoaderFlags; + uint32 NumberOfRvaAndSizes; + IMAGE_DATA_DIRECTORY DataDirectory[16]; +} IMAGE_OPTIONAL_HEADER; + +static int +match8(void *buf, char *cmp) +{ + return strncmp((char*)buf, cmp, 8) == 0; +} + +/* TODO(czaplinski): 64b windows? */ +/* + * Read from Windows PE/COFF .exe file image. + */ +static int +pedotout(int fd, Fhdr *fp, ExecHdr *hp) +{ + uint32 start, magic; + uint32 symtab, esymtab; + IMAGE_FILE_HEADER fh; + IMAGE_SECTION_HEADER sh; + IMAGE_OPTIONAL_HEADER oh; + uint8 sym[18]; + uint32 *valp; + int i; + + USED(hp); + seek(fd, 0x3c, 0); + if (readn(fd, &start, sizeof(start)) != sizeof(start)) { + werrstr("crippled PE MSDOS header"); + return 0; + } + start = leswal(start); + + seek(fd, start, 0); + if (readn(fd, &magic, sizeof(magic)) != sizeof(magic)) { + werrstr("no PE magic number found"); + return 0; + } + if (beswal(magic) != 0x50450000) { /* "PE\0\0" */ + werrstr("incorrect PE magic number"); + return 0; + } + + if (readn(fd, &fh, sizeof(fh)) != sizeof(fh)) { + werrstr("crippled PE File Header"); + return 0; + } + if (fh.PointerToSymbolTable == 0) { + werrstr("zero pointer to COFF symbol table"); + return 0; + } + + if (readn(fd, &oh, sizeof(oh)) != sizeof(oh)) { + werrstr("crippled PE Optional Header"); + return 0; + } + + seek(fd, start+sizeof(magic)+sizeof(fh)+leswab(fh.SizeOfOptionalHeader), 0); + fp->txtaddr = fp->dataddr = 0; + for (i=0; itxtaddr==0 || fp->dataddr==0) { + werrstr("no .text or .data"); + return 0; + } + + seek(fd, leswal(fh.PointerToSymbolTable), 0); + symtab = esymtab = 0; + for (i=0; i #include "obj.h" -int _is2(char* x) { return 0; } -int _is7(char* x) { return 0; } -int _is9(char* x) { return 0; } -int _isk(char* x) { return 0; } -int _isq(char* x) { return 0; } -int _isv(char* x) { return 0; } -int _isu(char* x) { return 0; } -int _read2(Biobuf* b, Prog* p) { return 0; } -int _read7(Biobuf* b, Prog* p) { return 0; } -int _read9(Biobuf* b, Prog* p) { return 0; } -int _readk(Biobuf* b, Prog* p) { return 0; } -int _readq(Biobuf* b, Prog* p) { return 0; } -int _readv(Biobuf* b, Prog* p) { return 0; } -int _readu(Biobuf* b, Prog* p) { return 0; } +int _is2(char* x) { USED(x); return 0; } +int _is7(char* x) { USED(x); return 0; } +int _is9(char* x) { USED(x); return 0; } +int _isk(char* x) { USED(x); return 0; } +int _isq(char* x) { USED(x); return 0; } +int _isv(char* x) { USED(x); return 0; } +int _isu(char* x) { USED(x); return 0; } +int _read2(Biobuf* b, Prog* p) { USED(b); USED(p); return 0; } +int _read7(Biobuf* b, Prog* p) { USED(b); USED(p); return 0; } +int _read9(Biobuf* b, Prog* p) { USED(b); USED(p); return 0; } +int _readk(Biobuf* b, Prog* p) { USED(b); USED(p); return 0; } +int _readq(Biobuf* b, Prog* p) { USED(b); USED(p); return 0; } +int _readv(Biobuf* b, Prog* p) { USED(b); USED(p); return 0; } +int _readu(Biobuf* b, Prog* p) { USED(b); USED(p); return 0; } diff --git a/src/libmach/linux.c b/src/libmach/linux.c index 6ce18957f..2c143266a 100644 --- a/src/libmach/linux.c +++ b/src/libmach/linux.c @@ -807,6 +807,8 @@ ptraceerr: static int ptracesegrw(Map *map, Seg *seg, uvlong addr, void *v, uint n, int isr) { + USED(seg); + return ptracerw(isr ? PTRACE_PEEKDATA : PTRACE_POKEDATA, PTRACE_PEEKDATA, isr, map->pid, addr, v, n); } @@ -937,6 +939,8 @@ ptraceregrw(Map *map, Seg *seg, uvlong addr, void *v, uint n, int isr) { int laddr; uvlong u; + + USED(seg); if((laddr = go2linux(addr)) < 0){ if(isr){ diff --git a/src/libmach/map.c b/src/libmach/map.c index ebfe03702..cd5ef0985 100644 --- a/src/libmach/map.c +++ b/src/libmach/map.c @@ -137,6 +137,8 @@ int fdrw(Map *map, Seg *s, uvlong addr, void *v, uint n, int isread) { int tot, m; + + USED(map); for(tot=0; tot 0 && line[len(line)-1] == '\r' { + // Put the '\r' back on buf and drop it from line. + // Let the next call to ReadLine check for "\r\n". + if b.r == 0 { + // should be unreachable + panic("bufio: tried to rewind past start of buffer") + } + b.r-- + line = line[:len(line)-1] + } return line, true, nil } @@ -307,10 +318,11 @@ func (b *Reader) ReadLine() (line []byte, isPrefix bool, err os.Error) { err = nil if line[len(line)-1] == '\n' { - line = line[:len(line)-1] - } - if len(line) > 0 && line[len(line)-1] == '\r' { - line = line[:len(line)-1] + drop := 1 + if len(line) > 1 && line[len(line)-2] == '\r' { + drop = 2 + } + line = line[:len(line)-drop] } return } diff --git a/src/pkg/bufio/bufio_test.go b/src/pkg/bufio/bufio_test.go index 82c73d36a..38213ffe7 100644 --- a/src/pkg/bufio/bufio_test.go +++ b/src/pkg/bufio/bufio_test.go @@ -137,7 +137,7 @@ var bufreaders = []bufReader{ } var bufsizes = []int{ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 2, 3, 4, 5, 6, 7, 8, 9, 10, 23, 32, 46, 64, 93, 128, 1024, 4096, } @@ -697,3 +697,71 @@ func TestLinesAfterRead(t *testing.T) { t.Errorf("expected EOF from ReadLine, got '%s' %t %s", line, isPrefix, err) } } + +type readLineResult struct { + line []byte + isPrefix bool + err os.Error +} + +var readLineNewlinesTests = []struct { + input string + bufSize int + expect []readLineResult +}{ + {"h\r\nb\r\n", 2, []readLineResult{ + {[]byte("h"), true, nil}, + {nil, false, nil}, + {[]byte("b"), true, nil}, + {nil, false, nil}, + {nil, false, os.EOF}, + }}, + {"hello\r\nworld\r\n", 6, []readLineResult{ + {[]byte("hello"), true, nil}, + {nil, false, nil}, + {[]byte("world"), true, nil}, + {nil, false, nil}, + {nil, false, os.EOF}, + }}, + {"hello\rworld\r", 6, []readLineResult{ + {[]byte("hello"), true, nil}, + {[]byte("\rworld"), true, nil}, + {[]byte("\r"), false, nil}, + {nil, false, os.EOF}, + }}, + {"h\ri\r\n\r", 2, []readLineResult{ + {[]byte("h"), true, nil}, + {[]byte("\ri"), true, nil}, + {nil, false, nil}, + {[]byte("\r"), false, nil}, + {nil, false, os.EOF}, + }}, +} + +func TestReadLineNewlines(t *testing.T) { + for _, e := range readLineNewlinesTests { + testReadLineNewlines(t, e.input, e.bufSize, e.expect) + } +} + +func testReadLineNewlines(t *testing.T, input string, bufSize int, expect []readLineResult) { + b, err := NewReaderSize(strings.NewReader(input), bufSize) + if err != nil { + t.Fatal(err) + } + for i, e := range expect { + line, isPrefix, err := b.ReadLine() + if bytes.Compare(line, e.line) != 0 { + t.Errorf("%q call %d, line == %q, want %q", input, i, line, e.line) + return + } + if isPrefix != e.isPrefix { + t.Errorf("%q call %d, isPrefix == %v, want %v", input, i, isPrefix, e.isPrefix) + return + } + if err != e.err { + t.Errorf("%q call %d, err == %v, want %v", input, i, err, e.err) + return + } + } +} diff --git a/src/pkg/bytes/buffer.go b/src/pkg/bytes/buffer.go index 5de86105d..975031bfa 100644 --- a/src/pkg/bytes/buffer.go +++ b/src/pkg/bytes/buffer.go @@ -336,13 +336,18 @@ func (b *Buffer) ReadString(delim byte) (line string, err os.Error) { // NewBuffer creates and initializes a new Buffer using buf as its initial // contents. It is intended to prepare a Buffer to read existing data. It -// can also be used to size the internal buffer for writing. To do that, +// can also be used to size the internal buffer for writing. To do that, // buf should have the desired capacity but a length of zero. +// +// In most cases, new(Buffer) (or just declaring a Buffer variable) is +// preferable to NewBuffer. In particular, passing a non-empty buf to +// NewBuffer and then writing to the Buffer will overwrite buf, not append to +// it. func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} } // NewBufferString creates and initializes a new Buffer using string s as its // initial contents. It is intended to prepare a buffer to read an existing -// string. +// string. See the warnings about NewBuffer; similar issues apply here. func NewBufferString(s string) *Buffer { return &Buffer{buf: []byte(s)} } diff --git a/src/pkg/container/heap/heap_test.go b/src/pkg/container/heap/heap_test.go index c5c1f76e1..6625e3a2b 100644 --- a/src/pkg/container/heap/heap_test.go +++ b/src/pkg/container/heap/heap_test.go @@ -6,32 +6,46 @@ package heap_test import ( "testing" - "container/vector" . "container/heap" ) -type myHeap struct { - // A vector.Vector implements sort.Interface except for Less, - // and it implements Push and Pop as required for heap.Interface. - vector.Vector +type myHeap []int + +func (h *myHeap) Less(i, j int) bool { + return (*h)[i] < (*h)[j] +} + +func (h *myHeap) Swap(i, j int) { + (*h)[i], (*h)[j] = (*h)[j], (*h)[i] +} + +func (h *myHeap) Len() int { + return len(*h) +} + +func (h *myHeap) Pop() (v interface{}) { + *h, v = (*h)[:h.Len()-1], (*h)[h.Len()-1] + return } -func (h *myHeap) Less(i, j int) bool { return h.At(i).(int) < h.At(j).(int) } +func (h *myHeap) Push(v interface{}) { + *h = append(*h, v.(int)) +} -func (h *myHeap) verify(t *testing.T, i int) { +func (h myHeap) verify(t *testing.T, i int) { n := h.Len() j1 := 2*i + 1 j2 := 2*i + 2 if j1 < n { if h.Less(j1, i) { - t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j1)) + t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h[i], j1, h[j1]) return } h.verify(t, j1) } if j2 < n { if h.Less(j2, i) { - t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h.At(i), j1, h.At(j2)) + t.Errorf("heap invariant invalidated [%d] = %d > [%d] = %d", i, h[i], j1, h[j2]) return } h.verify(t, j2) diff --git a/src/pkg/crypto/rand/rand_windows.go b/src/pkg/crypto/rand/rand_windows.go old mode 100755 new mode 100644 diff --git a/src/pkg/crypto/x509/verify_test.go b/src/pkg/crypto/x509/verify_test.go index 111f60eb1..ecff7ffd8 100644 --- a/src/pkg/crypto/x509/verify_test.go +++ b/src/pkg/crypto/x509/verify_test.go @@ -31,7 +31,7 @@ var verifyTests = []verifyTest{ dnsName: "www.google.com", expectedChains: [][]string{ - []string{"Google", "Thawte", "VeriSign"}, + {"Google", "Thawte", "VeriSign"}, }, }, { @@ -68,7 +68,7 @@ var verifyTests = []verifyTest{ dnsName: "www.google.com", expectedChains: [][]string{ - []string{"Google", "Thawte", "VeriSign"}, + {"Google", "Thawte", "VeriSign"}, }, }, { @@ -78,7 +78,7 @@ var verifyTests = []verifyTest{ currentTime: 1302726541, expectedChains: [][]string{ - []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, + {"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, }, }, { @@ -88,8 +88,8 @@ var verifyTests = []verifyTest{ currentTime: 1302726541, expectedChains: [][]string{ - []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, - []string{"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority", "StartCom Certification Authority"}, + {"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority"}, + {"dnssec-exp", "StartCom Class 1", "StartCom Certification Authority", "StartCom Certification Authority"}, }, }, } diff --git a/src/pkg/exp/norm/Makefile b/src/pkg/exp/norm/Makefile index a4dfb43f7..16239a72e 100644 --- a/src/pkg/exp/norm/Makefile +++ b/src/pkg/exp/norm/Makefile @@ -9,6 +9,7 @@ GOFILES=\ composition.go\ forminfo.go\ normalize.go\ + readwriter.go\ tables.go\ trie.go\ diff --git a/src/pkg/exp/norm/composition.go b/src/pkg/exp/norm/composition.go index b2d2abaf6..ea59c81cd 100644 --- a/src/pkg/exp/norm/composition.go +++ b/src/pkg/exp/norm/composition.go @@ -7,26 +7,25 @@ package norm import "utf8" const ( - maxCombiningChars = 30 + 2 // +2 to hold CGJ and Hangul overflow. + maxCombiningChars = 30 + maxBufferSize = maxCombiningChars + 2 // +1 to hold starter +1 to hold CGJ maxBackRunes = maxCombiningChars - 1 maxNFCExpansion = 3 // NFC(0x1D160) maxNFKCExpansion = 18 // NFKC(0xFDFA) - maxRuneSizeInDecomp = 4 - // Need to multiply by 2 as we don't reuse byte buffer space for recombining. - maxByteBufferSize = 2 * maxRuneSizeInDecomp * maxCombiningChars // 256 + maxByteBufferSize = utf8.UTFMax * maxBufferSize // 128 ) // reorderBuffer is used to normalize a single segment. Characters inserted with -// insert() are decomposed and reordered based on CCC. The compose() method can +// insert are decomposed and reordered based on CCC. The compose method can // be used to recombine characters. Note that the byte buffer does not hold // the UTF-8 characters in order. Only the rune array is maintained in sorted -// order. flush() writes the resulting segment to a byte array. +// order. flush writes the resulting segment to a byte array. type reorderBuffer struct { - rune [maxCombiningChars]runeInfo // Per character info. - byte [maxByteBufferSize]byte // UTF-8 buffer. Referenced by runeInfo.pos. - nrune int // Number of runeInfos. - nbyte uint8 // Number or bytes. + rune [maxBufferSize]runeInfo // Per character info. + byte [maxByteBufferSize]byte // UTF-8 buffer. Referenced by runeInfo.pos. + nrune int // Number of runeInfos. + nbyte uint8 // Number or bytes. f formInfo } @@ -49,10 +48,10 @@ func (rb *reorderBuffer) flush(out []byte) []byte { // insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class. // It returns false if the buffer is not large enough to hold the rune. -// It is used internally by insert. +// It is used internally by insert and insertString only. func (rb *reorderBuffer) insertOrdered(info runeInfo) bool { n := rb.nrune - if n >= maxCombiningChars { + if n >= maxCombiningChars+1 { return false } b := rb.rune[:] @@ -68,7 +67,7 @@ func (rb *reorderBuffer) insertOrdered(info runeInfo) bool { } rb.nrune += 1 pos := uint8(rb.nbyte) - rb.nbyte += info.size + rb.nbyte += utf8.UTFMax info.pos = pos b[n] = info return true @@ -81,17 +80,21 @@ func (rb *reorderBuffer) insert(src []byte, info runeInfo) bool { rune, _ := utf8.DecodeRune(src) return rb.decomposeHangul(uint32(rune)) } - pos := rb.nbyte if info.flags.hasDecomposition() { dcomp := rb.f.decompose(src) - for i := 0; i < len(dcomp); i += int(info.size) { + for i := 0; i < len(dcomp); { info = rb.f.info(dcomp[i:]) + pos := rb.nbyte if !rb.insertOrdered(info) { return false } + end := i + int(info.size) + copy(rb.byte[pos:], dcomp[i:end]) + i = end } - copy(rb.byte[pos:], dcomp) } else { + // insertOrder changes nbyte + pos := rb.nbyte if !rb.insertOrdered(info) { return false } @@ -107,18 +110,21 @@ func (rb *reorderBuffer) insertString(src string, info runeInfo) bool { rune, _ := utf8.DecodeRuneInString(src) return rb.decomposeHangul(uint32(rune)) } - pos := rb.nbyte - dcomp := rb.f.decomposeString(src) - dn := len(dcomp) - if dn != 0 { - for i := 0; i < dn; i += int(info.size) { + if info.flags.hasDecomposition() { + dcomp := rb.f.decomposeString(src) + for i := 0; i < len(dcomp); { info = rb.f.info(dcomp[i:]) + pos := rb.nbyte if !rb.insertOrdered(info) { return false } + end := i + int(info.size) + copy(rb.byte[pos:], dcomp[i:end]) + i = end } - copy(rb.byte[pos:], dcomp) } else { + // insertOrder changes nbyte + pos := rb.nbyte if !rb.insertOrdered(info) { return false } @@ -131,17 +137,16 @@ func (rb *reorderBuffer) insertString(src string, info runeInfo) bool { func (rb *reorderBuffer) appendRune(rune uint32) { bn := rb.nbyte sz := utf8.EncodeRune(rb.byte[bn:], int(rune)) - rb.nbyte += uint8(sz) + rb.nbyte += utf8.UTFMax rb.rune[rb.nrune] = runeInfo{bn, uint8(sz), 0, 0} rb.nrune++ } // assignRune sets a rune at position pos. It is used for Hangul and recomposition. func (rb *reorderBuffer) assignRune(pos int, rune uint32) { - bn := rb.nbyte + bn := rb.rune[pos].pos sz := utf8.EncodeRune(rb.byte[bn:], int(rune)) rb.rune[pos] = runeInfo{bn, uint8(sz), 0, 0} - rb.nbyte += uint8(sz) } // runeAt returns the rune at position n. It is used for Hangul and recomposition. @@ -259,11 +264,10 @@ func (rb *reorderBuffer) decomposeHangul(rune uint32) bool { // combineHangul algorithmically combines Jamo character components into Hangul. // See http://unicode.org/reports/tr15/#Hangul for details on combining Hangul. -func (rb *reorderBuffer) combineHangul() { - k := 1 +func (rb *reorderBuffer) combineHangul(s, i, k int) { b := rb.rune[:] bn := rb.nrune - for s, i := 0, 1; i < bn; i++ { + for ; i < bn; i++ { cccB := b[k-1].ccc cccC := b[i].ccc if cccB == 0 { @@ -305,14 +309,17 @@ func (rb *reorderBuffer) compose() { // blocked from S if and only if there is some character B between S // and C, and either B is a starter or it has the same or higher // combining class as C." + bn := rb.nrune + if bn == 0 { + return + } k := 1 b := rb.rune[:] - bn := rb.nrune for s, i := 0, 1; i < bn; i++ { if isJamoVT(rb.bytesAt(i)) { // Redo from start in Hangul mode. Necessary to support // U+320E..U+321E in NFKC mode. - rb.combineHangul() + rb.combineHangul(s, i, k) return } ii := b[i] diff --git a/src/pkg/exp/norm/forminfo.go b/src/pkg/exp/norm/forminfo.go index ee3edb8ea..5e01e89d1 100644 --- a/src/pkg/exp/norm/forminfo.go +++ b/src/pkg/exp/norm/forminfo.go @@ -77,7 +77,7 @@ func decompBoundary(f *formInfo, info runeInfo) bool { } func compBoundaryBefore(f *formInfo, info runeInfo) bool { - if info.ccc == 0 && info.flags.isYesC() { + if info.ccc == 0 && !info.flags.combinesBackward() { return true } // We assume that the CCC of the first character in a decomposition @@ -89,9 +89,7 @@ func compBoundaryBefore(f *formInfo, info runeInfo) bool { func compBoundaryAfter(f *formInfo, info runeInfo) bool { // This misses values where the last char in a decomposition is a // boundary such as Hangul with JamoT. - // TODO(mpvl): verify this does not lead to segments that do - // not fit in the reorderBuffer. - return info.flags.isInert() + return info.isInert() } // We pack quick check data in 4 bits: @@ -110,12 +108,15 @@ func (i qcInfo) isNoC() bool { return i&0x6 == 0x2 } func (i qcInfo) isMaybe() bool { return i&0x4 != 0 } func (i qcInfo) isYesD() bool { return i&0x1 == 0 } func (i qcInfo) isNoD() bool { return i&0x1 != 0 } -func (i qcInfo) isInert() bool { return i&0xf == 0 } func (i qcInfo) combinesForward() bool { return i&0x8 != 0 } func (i qcInfo) combinesBackward() bool { return i&0x4 != 0 } // == isMaybe func (i qcInfo) hasDecomposition() bool { return i&0x1 != 0 } // == isNoD +func (r runeInfo) isInert() bool { + return r.flags&0xf == 0 && r.ccc == 0 +} + // Wrappers for tables.go // The 16-bit value of the decompostion tries is an index into a byte diff --git a/src/pkg/exp/norm/maketables.go b/src/pkg/exp/norm/maketables.go index e3e5700a6..dbe560618 100644 --- a/src/pkg/exp/norm/maketables.go +++ b/src/pkg/exp/norm/maketables.go @@ -515,9 +515,13 @@ func completeCharFields(form int) { f.quickCheck[MComposed] = QCNo case (i & 0xffff00) == JamoLBase: f.quickCheck[MComposed] = QCYes - if JamoVBase <= i && i < JamoVEnd { + if JamoLBase <= i && i < JamoLEnd { + f.combinesForward = true + } + if JamoVBase <= i && i < JamoTEnd { f.quickCheck[MComposed] = QCMaybe f.combinesBackward = true + f.combinesForward = true } if JamoTBase <= i && i < JamoTEnd { f.quickCheck[MComposed] = QCMaybe @@ -562,7 +566,7 @@ func makeEntry(f *FormInfo) uint16 { case QCMaybe: e |= 0x6 default: - log.Fatalf("Illegal quickcheck value %d.", f.quickCheck[MComposed]) + log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed]) } return e } diff --git a/src/pkg/exp/norm/normalize.go b/src/pkg/exp/norm/normalize.go index e9d18dd9e..0bbf2547b 100644 --- a/src/pkg/exp/norm/normalize.go +++ b/src/pkg/exp/norm/normalize.go @@ -5,6 +5,8 @@ // Package norm contains types and functions for normalizing Unicode strings. package norm +import "utf8" + // A Form denotes a canonical representation of Unicode code points. // The Unicode-defined normalization and equivalence forms are: // @@ -32,17 +34,57 @@ const ( // Bytes returns f(b). May return b if f(b) = b. func (f Form) Bytes(b []byte) []byte { - panic("not implemented") + n := f.QuickSpan(b) + if n == len(b) { + return b + } + out := make([]byte, n, len(b)) + copy(out, b[0:n]) + return f.Append(out, b[n:]...) } // String returns f(s). func (f Form) String(s string) string { - panic("not implemented") + n := f.QuickSpanString(s) + if n == len(s) { + return s + } + out := make([]byte, 0, len(s)) + copy(out, s[0:n]) + return string(f.AppendString(out, s[n:])) } // IsNormal returns true if b == f(b). func (f Form) IsNormal(b []byte) bool { - panic("not implemented") + fd := formTable[f] + bp := quickSpan(fd, b) + if bp == len(b) { + return true + } + rb := reorderBuffer{f: *fd} + for bp < len(b) { + decomposeSegment(&rb, b[bp:]) + if fd.composing { + rb.compose() + } + for i := 0; i < rb.nrune; i++ { + info := rb.rune[i] + if bp+int(info.size) > len(b) { + return false + } + p := info.pos + pe := p + info.size + for ; p < pe; p++ { + if b[bp] != rb.byte[p] { + return false + } + bp++ + } + } + rb.reset() + bp += quickSpan(fd, b[bp:]) + } + return true } // IsNormalString returns true if s == f(s). @@ -50,14 +92,99 @@ func (f Form) IsNormalString(s string) bool { panic("not implemented") } +// patchTail fixes a case where a rune may be incorrectly normalized +// if it is followed by illegal continuation bytes. It returns the +// patched buffer and the number of trailing continuation bytes that +// have been dropped. +func patchTail(rb *reorderBuffer, buf []byte) ([]byte, int) { + info, p := lastRuneStart(&rb.f, buf) + if p == -1 || info.size == 0 { + return buf, 0 + } + end := p + int(info.size) + extra := len(buf) - end + if extra > 0 { + buf = decomposeToLastBoundary(rb, buf[:end]) + if rb.f.composing { + rb.compose() + } + return rb.flush(buf), extra + } + return buf, 0 +} + +func appendQuick(f *formInfo, dst, src []byte) ([]byte, int) { + if len(src) == 0 { + return dst, 0 + } + end := quickSpan(f, src) + return append(dst, src[:end]...), end +} + // Append returns f(append(out, b...)). -// The buffer out must be empty or equal to f(out). -func (f Form) Append(out, b []byte) []byte { - panic("not implemented") +// The buffer out must be nil, empty, or equal to f(out). +func (f Form) Append(out []byte, src ...byte) []byte { + if len(src) == 0 { + return out + } + fd := formTable[f] + rb := &reorderBuffer{f: *fd} + return doAppend(rb, out, src) +} + +func doAppend(rb *reorderBuffer, out, src []byte) []byte { + doMerge := len(out) > 0 + p := 0 + if !utf8.RuneStart(src[0]) { + // Move leading non-starters to destination. + for p++; p < len(src) && !utf8.RuneStart(src[p]); p++ { + } + out = append(out, src[:p]...) + buf, ndropped := patchTail(rb, out) + if ndropped > 0 { + out = append(buf, src[p-ndropped:p]...) + doMerge = false // no need to merge, ends with illegal UTF-8 + } else { + out = decomposeToLastBoundary(rb, buf) // force decomposition + } + } + fd := &rb.f + if doMerge { + var info runeInfo + if p < len(src) { + info = fd.info(src[p:]) + if p == 0 && !fd.boundaryBefore(fd, info) { + out = decomposeToLastBoundary(rb, out) + } + } + if info.size == 0 || fd.boundaryBefore(fd, info) { + if fd.composing { + rb.compose() + } + out = rb.flush(out) + if info.size == 0 { + // Append incomplete UTF-8 encoding. + return append(out, src[p:]...) + } + } + } + if rb.nrune == 0 { + src = src[p:] + out, p = appendQuick(fd, out, src) + } + for n := 0; p < len(src); p += n { + p += decomposeSegment(rb, src[p:]) + if fd.composing { + rb.compose() + } + out = rb.flush(out) + out, n = appendQuick(fd, out, src[p:]) + } + return out } // AppendString returns f(append(out, []byte(s))). -// The buffer out must be empty or equal to f(out). +// The buffer out must be nil, empty, or equal to f(out). func (f Form) AppendString(out []byte, s string) []byte { panic("not implemented") } @@ -65,7 +192,64 @@ func (f Form) AppendString(out []byte, s string) []byte { // QuickSpan returns a boundary n such that b[0:n] == f(b[0:n]). // It is not guaranteed to return the largest such n. func (f Form) QuickSpan(b []byte) int { - panic("not implemented") + return quickSpan(formTable[f], b) +} + +func quickSpan(fd *formInfo, b []byte) int { + var lastCC uint8 + var lastSegStart int + var i, nc int + for i < len(b) { + if b[i] < utf8.RuneSelf { + // Keep the loop tight for ASCII processing, as this is where + // most of the time is spent for this case. + for i++; i < len(b) && b[i] < utf8.RuneSelf; i++ { + } + lastSegStart = i - 1 + lastCC = 0 + nc = 0 + continue + } + info := fd.info(b[i:]) + if info.size == 0 { + // include incomplete runes + return len(b) + } + cc := info.ccc + if fd.composing { + if !info.flags.isYesC() { + break + } + } else { + if !info.flags.isYesD() { + break + } + } + if cc == 0 { + lastSegStart = i + nc = 0 + } else { + if nc >= maxCombiningChars { + lastSegStart = i + lastCC = cc + nc = 1 + } else { + if lastCC > cc { + return lastSegStart + } + nc++ + } + } + lastCC = cc + i += int(info.size) + } + if i == len(b) { + return len(b) + } + if fd.composing { + return lastSegStart + } + return i } // QuickSpanString returns a boundary n such that b[0:n] == f(s[0:n]). @@ -74,26 +258,174 @@ func (f Form) QuickSpanString(s string) int { panic("not implemented") } -// FirstBoundary returns the position i of the first boundary in b. -// It returns len(b), false if b contains no boundaries. -func (f Form) FirstBoundary(b []byte) (i int, ok bool) { - panic("not implemented") +// FirstBoundary returns the position i of the first boundary in b +// or -1 if b contains no boundary. +func (f Form) FirstBoundary(b []byte) int { + i := 0 + for ; i < len(b) && !utf8.RuneStart(b[i]); i++ { + } + if i >= len(b) { + return -1 + } + fd := formTable[f] + info := fd.info(b[i:]) + for n := 0; info.size != 0 && !fd.boundaryBefore(fd, info); { + i += int(info.size) + if n++; n >= maxCombiningChars { + return i + } + if i >= len(b) { + if !fd.boundaryAfter(fd, info) { + return -1 + } + return len(b) + } + info = fd.info(b[i:]) + } + if info.size == 0 { + return -1 + } + return i } -// FirstBoundaryInString return the position i of the first boundary in s. -// It returns len(s), false if s contains no boundaries. +// FirstBoundaryInString returns the position i of the first boundary in s +// or -1 if s contains no boundary. func (f Form) FirstBoundaryInString(s string) (i int, ok bool) { panic("not implemented") } -// LastBoundaryIn returns the position i of the last boundary in b. -// It returns 0, false if b contains no boundary. -func (f Form) LastBoundary(b []byte) (i int, ok bool) { - panic("not implemented") +// LastBoundary returns the position i of the last boundary in b +// or -1 if b contains no boundary. +func (f Form) LastBoundary(b []byte) int { + return lastBoundary(formTable[f], b) +} + +func lastBoundary(fd *formInfo, b []byte) int { + i := len(b) + info, p := lastRuneStart(fd, b) + if p == -1 { + return -1 + } + if info.size == 0 { // ends with incomplete rune + if p == 0 { // starts wtih incomplete rune + return -1 + } + i = p + info, p = lastRuneStart(fd, b[:i]) + if p == -1 { // incomplete UTF-8 encoding or non-starter bytes without a starter + return i + } + } + if p+int(info.size) != i { // trailing non-starter bytes: illegal UTF-8 + return i + } + if fd.boundaryAfter(fd, info) { + return i + } + i = p + for n := 0; i >= 0 && !fd.boundaryBefore(fd, info); { + info, p = lastRuneStart(fd, b[:i]) + if n++; n >= maxCombiningChars { + return len(b) + } + if p+int(info.size) != i { + if p == -1 { // no boundary found + return -1 + } + return i // boundary after an illegal UTF-8 encoding + } + i = p + } + return i } -// LastBoundaryInString returns the position i of the last boundary in s. -// It returns 0, false if s contains no boundary. -func (f Form) LastBoundaryInString(s string) (i int, ok bool) { +// LastBoundaryInString returns the position i of the last boundary in s +// or -1 if s contains no boundary. +func (f Form) LastBoundaryInString(s string) int { panic("not implemented") } + +// decomposeSegment scans the first segment in src into rb. +// It returns the number of bytes consumed from src. +// TODO(mpvl): consider inserting U+034f (Combining Grapheme Joiner) +// when we detect a sequence of 30+ non-starter chars. +func decomposeSegment(rb *reorderBuffer, src []byte) int { + // Force one character to be consumed. + info := rb.f.info(src) + if info.size == 0 { + return 0 + } + sp := 0 + for rb.insert(src[sp:], info) { + sp += int(info.size) + if sp >= len(src) { + break + } + info = rb.f.info(src[sp:]) + bound := rb.f.boundaryBefore(&rb.f, info) + if bound || info.size == 0 { + break + } + } + return sp +} + +// lastRuneStart returns the runeInfo and position of the last +// rune in buf or the zero runeInfo and -1 if no rune was found. +func lastRuneStart(fd *formInfo, buf []byte) (runeInfo, int) { + p := len(buf) - 1 + for ; p >= 0 && !utf8.RuneStart(buf[p]); p-- { + } + if p < 0 { + return runeInfo{0, 0, 0, 0}, -1 + } + return fd.info(buf[p:]), p +} + +// decomposeToLastBoundary finds an open segment at the end of the buffer +// and scans it into rb. Returns the buffer minus the last segment. +func decomposeToLastBoundary(rb *reorderBuffer, buf []byte) []byte { + fd := &rb.f + info, i := lastRuneStart(fd, buf) + if int(info.size) != len(buf)-i { + // illegal trailing continuation bytes + return buf + } + if rb.f.boundaryAfter(fd, info) { + return buf + } + var add [maxBackRunes]runeInfo // stores runeInfo in reverse order + add[0] = info + padd := 1 + n := 1 + p := len(buf) - int(info.size) + for ; p >= 0 && !rb.f.boundaryBefore(fd, info); p -= int(info.size) { + info, i = lastRuneStart(fd, buf[:p]) + if int(info.size) != p-i { + break + } + // Check that decomposition doesn't result in overflow. + if info.flags.hasDecomposition() { + dcomp := rb.f.decompose(buf[p-int(info.size):]) + for i := 0; i < len(dcomp); { + inf := rb.f.info(dcomp[i:]) + i += int(inf.size) + n++ + } + } else { + n++ + } + if n > maxBackRunes { + break + } + add[padd] = info + padd++ + } + pp := p + for padd--; padd >= 0; padd-- { + info = add[padd] + rb.insert(buf[pp:], info) + pp += int(info.size) + } + return buf[:p] +} diff --git a/src/pkg/exp/norm/normalize_test.go b/src/pkg/exp/norm/normalize_test.go new file mode 100644 index 000000000..6e8650d59 --- /dev/null +++ b/src/pkg/exp/norm/normalize_test.go @@ -0,0 +1,624 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import ( + "strings" + "testing" +) + +type PositionTest struct { + input string + pos int + buffer string // expected contents of reorderBuffer, if applicable +} + +type positionFunc func(rb *reorderBuffer, s string) int + +func runPosTests(t *testing.T, name string, f Form, fn positionFunc, tests []PositionTest) { + rb := reorderBuffer{f: *formTable[f]} + for i, test := range tests { + rb.reset() + pos := fn(&rb, test.input) + if pos != test.pos { + t.Errorf("%s:%d: position is %d; want %d", name, i, pos, test.pos) + } + runes := []int(test.buffer) + if rb.nrune != len(runes) { + t.Errorf("%s:%d: reorder buffer lenght is %d; want %d", name, i, rb.nrune, len(runes)) + continue + } + for j, want := range runes { + found := int(rb.runeAt(j)) + if found != want { + t.Errorf("%s:%d: rune at %d is %U; want %U", name, i, j, found, want) + } + } + } +} + +var decomposeSegmentTests = []PositionTest{ + // illegal runes + {"\xC0", 0, ""}, + {"\u00E0\x80", 2, "\u0061\u0300"}, + // starter + {"a", 1, "a"}, + {"ab", 1, "a"}, + // starter + composing + {"a\u0300", 3, "a\u0300"}, + {"a\u0300b", 3, "a\u0300"}, + // with decomposition + {"\u00C0", 2, "A\u0300"}, + {"\u00C0b", 2, "A\u0300"}, + // long + {strings.Repeat("\u0300", 31), 62, strings.Repeat("\u0300", 31)}, + // ends with incomplete UTF-8 encoding + {"\xCC", 0, ""}, + {"\u0300\xCC", 2, "\u0300"}, +} + +func decomposeSegmentF(rb *reorderBuffer, s string) int { + return decomposeSegment(rb, []byte(s)) +} + +func TestDecomposeSegment(t *testing.T) { + runPosTests(t, "TestDecomposeSegment", NFC, decomposeSegmentF, decomposeSegmentTests) +} + +var firstBoundaryTests = []PositionTest{ + // no boundary + {"", -1, ""}, + {"\u0300", -1, ""}, + {"\x80\x80", -1, ""}, + // illegal runes + {"\xff", 0, ""}, + {"\u0300\xff", 2, ""}, + {"\u0300\xc0\x80\x80", 2, ""}, + // boundaries + {"a", 0, ""}, + {"\u0300a", 2, ""}, + // Hangul + {"\u1103\u1161", 0, ""}, + {"\u110B\u1173\u11B7", 0, ""}, + {"\u1161\u110B\u1173\u11B7", 3, ""}, + {"\u1173\u11B7\u1103\u1161", 6, ""}, + // too many combining characters. + {strings.Repeat("\u0300", maxCombiningChars-1), -1, ""}, + {strings.Repeat("\u0300", maxCombiningChars), 60, ""}, + {strings.Repeat("\u0300", maxCombiningChars+1), 60, ""}, +} + +func firstBoundary(rb *reorderBuffer, s string) int { + return rb.f.form.FirstBoundary([]byte(s)) +} + +func TestFirstBoundary(t *testing.T) { + runPosTests(t, "TestFirstBoundary", NFC, firstBoundary, firstBoundaryTests) +} + +var decomposeToLastTests = []PositionTest{ + // ends with inert character + {"Hello!", 6, ""}, + {"\u0632", 2, ""}, + {"a\u0301\u0635", 5, ""}, + // ends with non-inert starter + {"a", 0, "a"}, + {"a\u0301a", 3, "a"}, + {"a\u0301\u03B9", 3, "\u03B9"}, + {"a\u0327", 0, "a\u0327"}, + // illegal runes + {"\xFF", 1, ""}, + {"aa\xFF", 3, ""}, + {"\xC0\x80\x80", 3, ""}, + {"\xCC\x80\x80", 3, ""}, + // ends with incomplete UTF-8 encoding + {"a\xCC", 2, ""}, + // ends with combining characters + {"\u0300\u0301", 0, "\u0300\u0301"}, + {"a\u0300\u0301", 0, "a\u0300\u0301"}, + {"a\u0301\u0308", 0, "a\u0301\u0308"}, + {"a\u0308\u0301", 0, "a\u0308\u0301"}, + {"aaaa\u0300\u0301", 3, "a\u0300\u0301"}, + {"\u0300a\u0300\u0301", 2, "a\u0300\u0301"}, + {"\u00C0", 0, "A\u0300"}, + {"a\u00C0", 1, "A\u0300"}, + // decomposing + {"a\u0300\uFDC0", 3, "\u0645\u062C\u064A"}, + {"\uFDC0" + strings.Repeat("\u0300", 26), 0, "\u0645\u062C\u064A" + strings.Repeat("\u0300", 26)}, + // Hangul + {"a\u1103", 1, "\u1103"}, + {"a\u110B", 1, "\u110B"}, + {"a\u110B\u1173", 1, "\u110B\u1173"}, + // See comment in composition.go:compBoundaryAfter. + {"a\u110B\u1173\u11B7", 1, "\u110B\u1173\u11B7"}, + {"a\uC73C", 1, "\u110B\u1173"}, + {"다음", 3, "\u110B\u1173\u11B7"}, + {"다", 0, "\u1103\u1161"}, + {"\u1103\u1161\u110B\u1173\u11B7", 6, "\u110B\u1173\u11B7"}, + {"\u110B\u1173\u11B7\u1103\u1161", 9, "\u1103\u1161"}, + {"다음음", 6, "\u110B\u1173\u11B7"}, + {"음다다", 6, "\u1103\u1161"}, + // buffer overflow + {"a" + strings.Repeat("\u0300", 30), 3, strings.Repeat("\u0300", 29)}, + {"\uFDFA" + strings.Repeat("\u0300", 14), 3, strings.Repeat("\u0300", 14)}, + // weird UTF-8 + {"a\u0300\u11B7", 0, "a\u0300\u11B7"}, +} + +func decomposeToLast(rb *reorderBuffer, s string) int { + buf := decomposeToLastBoundary(rb, []byte(s)) + return len(buf) +} + +func TestDecomposeToLastBoundary(t *testing.T) { + runPosTests(t, "TestDecomposeToLastBoundary", NFKC, decomposeToLast, decomposeToLastTests) +} + +var lastBoundaryTests = []PositionTest{ + // ends with inert character + {"Hello!", 6, ""}, + {"\u0632", 2, ""}, + // ends with non-inert starter + {"a", 0, ""}, + // illegal runes + {"\xff", 1, ""}, + {"aa\xff", 3, ""}, + {"a\xff\u0300", 1, ""}, + {"\xc0\x80\x80", 3, ""}, + {"\xc0\x80\x80\u0300", 3, ""}, + // ends with incomplete UTF-8 encoding + {"\xCC", -1, ""}, + {"\xE0\x80", -1, ""}, + {"\xF0\x80\x80", -1, ""}, + {"a\xCC", 0, ""}, + {"\x80\xCC", 1, ""}, + {"\xCC\xCC", 1, ""}, + // ends with combining characters + {"a\u0300\u0301", 0, ""}, + {"aaaa\u0300\u0301", 3, ""}, + {"\u0300a\u0300\u0301", 2, ""}, + {"\u00C0", 0, ""}, + {"a\u00C0", 1, ""}, + // decomposition may recombine + {"\u0226", 0, ""}, + // no boundary + {"", -1, ""}, + {"\u0300\u0301", -1, ""}, + {"\u0300", -1, ""}, + {"\x80\x80", -1, ""}, + {"\x80\x80\u0301", -1, ""}, + // Hangul + {"다음", 3, ""}, + {"다", 0, ""}, + {"\u1103\u1161\u110B\u1173\u11B7", 6, ""}, + {"\u110B\u1173\u11B7\u1103\u1161", 9, ""}, + // too many combining characters. + {strings.Repeat("\u0300", maxCombiningChars-1), -1, ""}, + {strings.Repeat("\u0300", maxCombiningChars), 60, ""}, + {strings.Repeat("\u0300", maxCombiningChars+1), 62, ""}, +} + +func lastBoundaryF(rb *reorderBuffer, s string) int { + return rb.f.form.LastBoundary([]byte(s)) +} + +func TestLastBoundary(t *testing.T) { + runPosTests(t, "TestLastBoundary", NFC, lastBoundaryF, lastBoundaryTests) +} + +var quickSpanTests = []PositionTest{ + {"", 0, ""}, + // starters + {"a", 1, ""}, + {"abc", 3, ""}, + {"\u043Eb", 3, ""}, + // incomplete last rune. + {"\xCC", 1, ""}, + {"a\xCC", 2, ""}, + // incorrectly ordered combining characters + {"\u0300\u0316", 0, ""}, + {"\u0300\u0316cd", 0, ""}, + // have a maximum number of combining characters. + {strings.Repeat("\u035D", 30) + "\u035B", 62, ""}, + {"a" + strings.Repeat("\u035D", 30) + "\u035B", 63, ""}, + {"Ɵ" + strings.Repeat("\u035D", 30) + "\u035B", 64, ""}, + {"aa" + strings.Repeat("\u035D", 30) + "\u035B", 64, ""}, +} + +var quickSpanNFDTests = []PositionTest{ + // needs decomposing + {"\u00C0", 0, ""}, + {"abc\u00C0", 3, ""}, + // correctly ordered combining characters + {"\u0300", 2, ""}, + {"ab\u0300", 4, ""}, + {"ab\u0300cd", 6, ""}, + {"\u0300cd", 4, ""}, + {"\u0316\u0300", 4, ""}, + {"ab\u0316\u0300", 6, ""}, + {"ab\u0316\u0300cd", 8, ""}, + {"ab\u0316\u0300\u00C0", 6, ""}, + {"\u0316\u0300cd", 6, ""}, + {"\u043E\u0308b", 5, ""}, + // incorrectly ordered combining characters + {"ab\u0300\u0316", 1, ""}, // TODO(mpvl): we could skip 'b' as well. + {"ab\u0300\u0316cd", 1, ""}, + // Hangul + {"같은", 0, ""}, +} + +var quickSpanNFCTests = []PositionTest{ + // okay composed + {"\u00C0", 2, ""}, + {"abc\u00C0", 5, ""}, + // correctly ordered combining characters + {"ab\u0300", 1, ""}, + {"ab\u0300cd", 1, ""}, + {"ab\u0316\u0300", 1, ""}, + {"ab\u0316\u0300cd", 1, ""}, + {"\u00C0\u035D", 4, ""}, + // we do not special case leading combining characters + {"\u0300cd", 0, ""}, + {"\u0300", 0, ""}, + {"\u0316\u0300", 0, ""}, + {"\u0316\u0300cd", 0, ""}, + // incorrectly ordered combining characters + {"ab\u0300\u0316", 1, ""}, + {"ab\u0300\u0316cd", 1, ""}, + // Hangul + {"같은", 6, ""}, +} + +func doQuickSpan(rb *reorderBuffer, s string) int { + return rb.f.form.QuickSpan([]byte(s)) +} + +func TestQuickSpan(t *testing.T) { + runPosTests(t, "TestQuickSpanNFD1", NFD, doQuickSpan, quickSpanTests) + runPosTests(t, "TestQuickSpanNFD2", NFD, doQuickSpan, quickSpanNFDTests) + runPosTests(t, "TestQuickSpanNFC1", NFC, doQuickSpan, quickSpanTests) + runPosTests(t, "TestQuickSpanNFC2", NFC, doQuickSpan, quickSpanNFCTests) +} + +var isNormalTests = []PositionTest{ + {"", 1, ""}, + // illegal runes + {"\xff", 1, ""}, + // starters + {"a", 1, ""}, + {"abc", 1, ""}, + {"\u043Eb", 1, ""}, + // incorrectly ordered combining characters + {"\u0300\u0316", 0, ""}, + {"ab\u0300\u0316", 0, ""}, + {"ab\u0300\u0316cd", 0, ""}, + {"\u0300\u0316cd", 0, ""}, +} +var isNormalNFDTests = []PositionTest{ + // needs decomposing + {"\u00C0", 0, ""}, + {"abc\u00C0", 0, ""}, + // correctly ordered combining characters + {"\u0300", 1, ""}, + {"ab\u0300", 1, ""}, + {"ab\u0300cd", 1, ""}, + {"\u0300cd", 1, ""}, + {"\u0316\u0300", 1, ""}, + {"ab\u0316\u0300", 1, ""}, + {"ab\u0316\u0300cd", 1, ""}, + {"\u0316\u0300cd", 1, ""}, + {"\u043E\u0308b", 1, ""}, + // Hangul + {"같은", 0, ""}, +} +var isNormalNFCTests = []PositionTest{ + // okay composed + {"\u00C0", 1, ""}, + {"abc\u00C0", 1, ""}, + // need reordering + {"a\u0300", 0, ""}, + {"a\u0300cd", 0, ""}, + {"a\u0316\u0300", 0, ""}, + {"a\u0316\u0300cd", 0, ""}, + // correctly ordered combining characters + {"ab\u0300", 1, ""}, + {"ab\u0300cd", 1, ""}, + {"ab\u0316\u0300", 1, ""}, + {"ab\u0316\u0300cd", 1, ""}, + {"\u00C0\u035D", 1, ""}, + {"\u0300", 1, ""}, + {"\u0316\u0300cd", 1, ""}, + // Hangul + {"같은", 1, ""}, +} + +func isNormal(rb *reorderBuffer, s string) int { + if rb.f.form.IsNormal([]byte(s)) { + return 1 + } + return 0 +} + +func TestIsNormal(t *testing.T) { + runPosTests(t, "TestIsNormalNFD1", NFD, isNormal, isNormalTests) + runPosTests(t, "TestIsNormalNFD2", NFD, isNormal, isNormalNFDTests) + runPosTests(t, "TestIsNormalNFC1", NFC, isNormal, isNormalTests) + runPosTests(t, "TestIsNormalNFC2", NFC, isNormal, isNormalNFCTests) +} + +type AppendTest struct { + left string + right string + out string +} + +type appendFunc func(f Form, out []byte, s string) []byte + +func runAppendTests(t *testing.T, name string, f Form, fn appendFunc, tests []AppendTest) { + for i, test := range tests { + out := []byte(test.left) + out = fn(f, out, test.right) + outs := string(out) + if len(outs) != len(test.out) { + t.Errorf("%s:%d: length is %d; want %d", name, i, len(outs), len(test.out)) + } + if outs != test.out { + // Find first rune that differs and show context. + ir := []int(outs) + ig := []int(test.out) + for j := 0; j < len(ir) && j < len(ig); j++ { + if ir[j] == ig[j] { + continue + } + if j -= 3; j < 0 { + j = 0 + } + for e := j + 7; j < e && j < len(ir) && j < len(ig); j++ { + t.Errorf("%s:%d: runeAt(%d) = %U; want %U", name, i, j, ir[j], ig[j]) + } + break + } + } + } +} + +var appendTests = []AppendTest{ + // empty buffers + {"", "", ""}, + {"a", "", "a"}, + {"", "a", "a"}, + {"", "\u0041\u0307\u0304", "\u01E0"}, + // segment split across buffers + {"", "a\u0300b", "\u00E0b"}, + {"a", "\u0300b", "\u00E0b"}, + {"a", "\u0300\u0316", "\u00E0\u0316"}, + {"a", "\u0316\u0300", "\u00E0\u0316"}, + {"a", "\u0300a\u0300", "\u00E0\u00E0"}, + {"a", "\u0300a\u0300a\u0300", "\u00E0\u00E0\u00E0"}, + {"a", "\u0300aaa\u0300aaa\u0300", "\u00E0aa\u00E0aa\u00E0"}, + {"a\u0300", "\u0327", "\u00E0\u0327"}, + {"a\u0327", "\u0300", "\u00E0\u0327"}, + {"a\u0316", "\u0300", "\u00E0\u0316"}, + {"\u0041\u0307", "\u0304", "\u01E0"}, + // Hangul + {"", "\u110B\u1173", "\uC73C"}, + {"", "\u1103\u1161", "\uB2E4"}, + {"", "\u110B\u1173\u11B7", "\uC74C"}, + {"", "\u320E", "\x28\uAC00\x29"}, + {"", "\x28\u1100\u1161\x29", "\x28\uAC00\x29"}, + {"\u1103", "\u1161", "\uB2E4"}, + {"\u110B", "\u1173\u11B7", "\uC74C"}, + {"\u110B\u1173", "\u11B7", "\uC74C"}, + {"\uC73C", "\u11B7", "\uC74C"}, + // UTF-8 encoding split across buffers + {"a\xCC", "\x80", "\u00E0"}, + {"a\xCC", "\x80b", "\u00E0b"}, + {"a\xCC", "\x80a\u0300", "\u00E0\u00E0"}, + {"a\xCC", "\x80\x80", "\u00E0\x80"}, + {"a\xCC", "\x80\xCC", "\u00E0\xCC"}, + {"a\u0316\xCC", "\x80a\u0316\u0300", "\u00E0\u0316\u00E0\u0316"}, + // ending in incomplete UTF-8 encoding + {"", "\xCC", "\xCC"}, + {"a", "\xCC", "a\xCC"}, + {"a", "b\xCC", "ab\xCC"}, + {"\u0226", "\xCC", "\u0226\xCC"}, + // illegal runes + {"", "\x80", "\x80"}, + {"", "\x80\x80\x80", "\x80\x80\x80"}, + {"", "\xCC\x80\x80\x80", "\xCC\x80\x80\x80"}, + {"", "a\x80", "a\x80"}, + {"", "a\x80\x80\x80", "a\x80\x80\x80"}, + {"", "a\x80\x80\x80\x80\x80\x80", "a\x80\x80\x80\x80\x80\x80"}, + {"a", "\x80\x80\x80", "a\x80\x80\x80"}, + // overflow + {"", strings.Repeat("\x80", 33), strings.Repeat("\x80", 33)}, + {strings.Repeat("\x80", 33), "", strings.Repeat("\x80", 33)}, + {strings.Repeat("\x80", 33), strings.Repeat("\x80", 33), strings.Repeat("\x80", 66)}, + // overflow of combining characters + {strings.Repeat("\u0300", 33), "", strings.Repeat("\u0300", 33)}, + // weird UTF-8 + {"\u00E0\xE1", "\x86", "\u00E0\xE1\x86"}, + {"a\u0300\u11B7", "\u0300", "\u00E0\u11B7\u0300"}, + {"a\u0300\u11B7\u0300", "\u0300", "\u00E0\u11B7\u0300\u0300"}, + {"\u0300", "\xF8\x80\x80\x80\x80\u0300", "\u0300\xF8\x80\x80\x80\x80\u0300"}, + {"\u0300", "\xFC\x80\x80\x80\x80\x80\u0300", "\u0300\xFC\x80\x80\x80\x80\x80\u0300"}, + {"\xF8\x80\x80\x80\x80\u0300", "\u0300", "\xF8\x80\x80\x80\x80\u0300\u0300"}, + {"\xFC\x80\x80\x80\x80\x80\u0300", "\u0300", "\xFC\x80\x80\x80\x80\x80\u0300\u0300"}, +} + +func appendF(f Form, out []byte, s string) []byte { + return f.Append(out, []byte(s)...) +} + +func TestAppend(t *testing.T) { + runAppendTests(t, "TestAppend", NFKC, appendF, appendTests) +} + +func doFormBenchmark(b *testing.B, f Form, s string) { + b.StopTimer() + in := []byte(s) + buf := make([]byte, 2*len(in)) + b.SetBytes(int64(len(s))) + b.StartTimer() + for i := 0; i < b.N; i++ { + buf = f.Append(buf[0:0], in...) + buf = buf[0:0] + } +} + +var ascii = strings.Repeat("There is nothing to change here! ", 500) + +func BenchmarkNormalizeAsciiNFC(b *testing.B) { + doFormBenchmark(b, NFC, ascii) +} +func BenchmarkNormalizeAsciiNFD(b *testing.B) { + doFormBenchmark(b, NFD, ascii) +} +func BenchmarkNormalizeAsciiNFKC(b *testing.B) { + doFormBenchmark(b, NFKC, ascii) +} +func BenchmarkNormalizeAsciiNFKD(b *testing.B) { + doFormBenchmark(b, NFKD, ascii) +} + +func doTextBenchmark(b *testing.B, s string) { + b.StopTimer() + in := make([]byte, len(s)) + for i := range s { + in[i] = s[i] + } + // Using copy(in, s) makes many tests much slower!? + b.SetBytes(int64(len(s)) * 4) + var buf = make([]byte, 2*len(in)) + b.StartTimer() + for i := 0; i < b.N; i++ { + buf = NFC.Append(buf[0:0], in...) + buf = NFD.Append(buf[0:0], in...) + buf = NFKC.Append(buf[0:0], in...) + buf = NFKD.Append(buf[0:0], in...) + } +} + +func BenchmarkCanonicalOrdering(b *testing.B) { + doTextBenchmark(b, txt_canon) +} +func BenchmarkExtendedLatin(b *testing.B) { + doTextBenchmark(b, txt_vn) +} +func BenchmarkMiscTwoByteUtf8(b *testing.B) { + doTextBenchmark(b, twoByteUtf8) +} +func BenchmarkMiscThreeByteUtf8(b *testing.B) { + doTextBenchmark(b, threeByteUtf8) +} +func BenchmarkHangul(b *testing.B) { + doTextBenchmark(b, txt_kr) +} +func BenchmarkJapanese(b *testing.B) { + doTextBenchmark(b, txt_jp) +} +func BenchmarkChinese(b *testing.B) { + doTextBenchmark(b, txt_cn) +} + +// Tests sampled from the Canonical ordering tests (Part 2) of +// http://unicode.org/Public/UNIDATA/NormalizationTest.txt +const txt_canon = `\u0061\u0315\u0300\u05AE\u0300\u0062 \u0061\u0300\u0315\u0300\u05AE\u0062 +\u0061\u0302\u0315\u0300\u05AE\u0062 \u0061\u0307\u0315\u0300\u05AE\u0062 +\u0061\u0315\u0300\u05AE\u030A\u0062 \u0061\u059A\u0316\u302A\u031C\u0062 +\u0061\u032E\u059A\u0316\u302A\u0062 \u0061\u0338\u093C\u0334\u0062 +\u0061\u059A\u0316\u302A\u0339 \u0061\u0341\u0315\u0300\u05AE\u0062 +\u0061\u0348\u059A\u0316\u302A\u0062 \u0061\u0361\u0345\u035D\u035C\u0062 +\u0061\u0366\u0315\u0300\u05AE\u0062 \u0061\u0315\u0300\u05AE\u0486\u0062 +\u0061\u05A4\u059A\u0316\u302A\u0062 \u0061\u0315\u0300\u05AE\u0613\u0062 +\u0061\u0315\u0300\u05AE\u0615\u0062 \u0061\u0617\u0315\u0300\u05AE\u0062 +\u0061\u0619\u0618\u064D\u064E\u0062 \u0061\u0315\u0300\u05AE\u0654\u0062 +\u0061\u0315\u0300\u05AE\u06DC\u0062 \u0061\u0733\u0315\u0300\u05AE\u0062 +\u0061\u0744\u059A\u0316\u302A\u0062 \u0061\u0315\u0300\u05AE\u0745\u0062 +\u0061\u09CD\u05B0\u094D\u3099\u0062 \u0061\u0E38\u0E48\u0E38\u0C56\u0062 +\u0061\u0EB8\u0E48\u0E38\u0E49\u0062 \u0061\u0F72\u0F71\u0EC8\u0F71\u0062 +\u0061\u1039\u05B0\u094D\u3099\u0062 \u0061\u05B0\u094D\u3099\u1A60\u0062 +\u0061\u3099\u093C\u0334\u1BE6\u0062 \u0061\u3099\u093C\u0334\u1C37\u0062 +\u0061\u1CD9\u059A\u0316\u302A\u0062 \u0061\u2DED\u0315\u0300\u05AE\u0062 +\u0061\u2DEF\u0315\u0300\u05AE\u0062 \u0061\u302D\u302E\u059A\u0316\u0062` + +// Taken from http://creativecommons.org/licenses/by-sa/3.0/vn/ +const txt_vn = `Với các điều kiện sau: Ghi nhận công của tác giả. +Nếu bạn sử dụng, chuyển đổi, hoặc xây dựng dự án từ +nội dung được chia sẻ này, bạn phải áp dụng giấy phép này hoặc +một giấy phép khác có các điều khoản tương tự như giấy phép này +cho dự án của bạn. Hiểu rằng: Miễn — Bất kỳ các điều kiện nào +trên đây cũng có thể được miễn bỏ nếu bạn được sự cho phép của +người sở hữu bản quyền. Phạm vi công chúng — Khi tác phẩm hoặc +bất kỳ chương nào của tác phẩm đã trong vùng dành cho công +chúng theo quy định của pháp luật thì tình trạng của nó không +bị ảnh hưởng bởi giấy phép trong bất kỳ trường hợp nào.` + +// Taken from http://creativecommons.org/licenses/by-sa/1.0/deed.ru +const txt_ru = `При обязательном соблюдении следующих условий: +Attribution — Вы должны атрибутировать произведение (указывать +автора и источник) в порядке, предусмотренном автором или +лицензиаром (но только так, чтобы никоим образом не подразумевалось, +что они поддерживают вас или использование вами данного произведения). +Υπό τις ακόλουθες προϋποθέσεις:` + +// Taken from http://creativecommons.org/licenses/by-sa/3.0/gr/ +const txt_gr = `Αναφορά Δημιουργού — Θα πρέπει να κάνετε την αναφορά στο έργο με τον +τρόπο που έχει οριστεί από το δημιουργό ή το χορηγούντο την άδεια +(χωρίς όμως να εννοείται με οποιονδήποτε τρόπο ότι εγκρίνουν εσάς ή +τη χρήση του έργου από εσάς). Παρόμοια Διανομή — Εάν αλλοιώσετε, +τροποποιήσετε ή δημιουργήσετε περαιτέρω βασισμένοι στο έργο θα +μπορείτε να διανέμετε το έργο που θα προκύψει μόνο με την ίδια ή +παρόμοια άδεια.` + +// Taken from http://creativecommons.org/licenses/by-sa/3.0/deed.ar +const txt_ar = `بموجب الشروط التالية نسب المصنف — يجب عليك أن +تنسب العمل بالطريقة التي تحددها المؤلف أو المرخص (ولكن ليس بأي حال من +الأحوال أن توحي وتقترح بتحول أو استخدامك للعمل). +المشاركة على قدم المساواة — إذا كنت يعدل ، والتغيير ، أو الاستفادة +من هذا العمل ، قد ينتج عن توزيع العمل إلا في ظل تشابه او تطابق فى واحد +لهذا الترخيص.` + +// Taken from http://creativecommons.org/licenses/by-sa/1.0/il/ +const txt_il = `בכפוף לתנאים הבאים: ייחוס — עליך לייחס את היצירה (לתת קרדיט) באופן +המצויין על-ידי היוצר או מעניק הרישיון (אך לא בשום אופן המרמז על כך +שהם תומכים בך או בשימוש שלך ביצירה). שיתוף זהה — אם תחליט/י לשנות, +לעבד או ליצור יצירה נגזרת בהסתמך על יצירה זו, תוכל/י להפיץ את יצירתך +החדשה רק תחת אותו הרישיון או רישיון דומה לרישיון זה.` + +const twoByteUtf8 = txt_ru + txt_gr + txt_ar + txt_il + +// Taken from http://creativecommons.org/licenses/by-sa/2.0/kr/ +const txt_kr = `다음과 같은 조건을 따라야 합니다: 저작자표시 +(Attribution) — 저작자나 이용허락자가 정한 방법으로 저작물의 +원저작자를 표시하여야 합니다(그러나 원저작자가 이용자나 이용자의 +이용을 보증하거나 추천한다는 의미로 표시해서는 안됩니다). +동일조건변경허락 — 이 저작물을 이용하여 만든 이차적 저작물에는 본 +라이선스와 동일한 라이선스를 적용해야 합니다.` + +// Taken from http://creativecommons.org/licenses/by-sa/3.0/th/ +const txt_th = `ภายใต้เงื่อนไข ดังต่อไปนี้ : แสดงที่มา — คุณต้องแสดงที่ +มาของงานดังกล่าว ตามรูปแบบที่ผู้สร้างสรรค์หรือผู้อนุญาตกำหนด (แต่ +ไม่ใช่ในลักษณะที่ว่า พวกเขาสนับสนุนคุณหรือสนับสนุนการที่ +คุณนำงานไปใช้) อนุญาตแบบเดียวกัน — หากคุณดัดแปลง เปลี่ยนรูป หรื +อต่อเติมงานนี้ คุณต้องใช้สัญญาอนุญาตแบบเดียวกันหรือแบบที่เหมื +อนกับสัญญาอนุญาตที่ใช้กับงานนี้เท่านั้น` + +const threeByteUtf8 = txt_th + +// Taken from http://creativecommons.org/licenses/by-sa/2.0/jp/ +const txt_jp = `あなたの従うべき条件は以下の通りです。 +表示 — あなたは原著作者のクレジットを表示しなければなりません。 +継承 — もしあなたがこの作品を改変、変形または加工した場合、 +あなたはその結果生じた作品をこの作品と同一の許諾条件の下でのみ +頒布することができます。` + +// http://creativecommons.org/licenses/by-sa/2.5/cn/ +const txt_cn = `您可以自由： 复制、发行、展览、表演、放映、 +广播或通过信息网络传播本作品 创作演绎作品 +对本作品进行商业性使用 惟须遵守下列条件： +署名 — 您必须按照作者或者许可人指定的方式对作品进行署名。 +相同方式共享 — 如果您改变、转换本作品或者以本作品为基础进行创作， +您只能采用与本协议相同的许可协议发布基于本作品的演绎作品。` diff --git a/src/pkg/exp/norm/readwriter.go b/src/pkg/exp/norm/readwriter.go new file mode 100644 index 000000000..fce6c4054 --- /dev/null +++ b/src/pkg/exp/norm/readwriter.go @@ -0,0 +1,121 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import ( + "io" + "os" +) + +type normWriter struct { + rb reorderBuffer + w io.Writer + buf []byte +} + +// Write implements the standard write interface. If the last characters are +// not at a normalization boundary, the bytes will be buffered for the next +// write. The remaining bytes will be written on close. +func (w *normWriter) Write(data []byte) (n int, err os.Error) { + // Process data in pieces to keep w.buf size bounded. + const chunk = 4000 + + for len(data) > 0 { + // Normalize into w.buf. + m := len(data) + if m > chunk { + m = chunk + } + w.buf = doAppend(&w.rb, w.buf, data[:m]) + data = data[m:] + n += m + + // Write out complete prefix, save remainder. + // Note that lastBoundary looks back at most 30 runes. + i := lastBoundary(&w.rb.f, w.buf) + if i == -1 { + i = 0 + } + if i > 0 { + if _, err = w.w.Write(w.buf[:i]); err != nil { + break + } + bn := copy(w.buf, w.buf[i:]) + w.buf = w.buf[:bn] + } + } + return n, err +} + +// Close forces data that remains in the buffer to be written. +func (w *normWriter) Close() os.Error { + if len(w.buf) > 0 { + _, err := w.w.Write(w.buf) + if err != nil { + return err + } + } + return nil +} + +// Writer returns a new writer that implements Write(b) +// by writing f(b) to w. The returned writer may use an +// an internal buffer to maintain state across Write calls. +// Calling its Close method writes any buffered data to w. +func (f Form) Writer(w io.Writer) io.WriteCloser { + return &normWriter{rb: reorderBuffer{f: *formTable[f]}, w: w} +} + +type normReader struct { + rb reorderBuffer + r io.Reader + inbuf []byte + outbuf []byte + bufStart int + lastBoundary int + err os.Error +} + +// Read implements the standard read interface. +func (r *normReader) Read(p []byte) (int, os.Error) { + for { + if r.lastBoundary-r.bufStart > 0 { + n := copy(p, r.outbuf[r.bufStart:r.lastBoundary]) + r.bufStart += n + if r.lastBoundary-r.bufStart > 0 { + return n, nil + } + return n, r.err + } + if r.err != nil { + return 0, r.err + } + outn := copy(r.outbuf, r.outbuf[r.lastBoundary:]) + r.outbuf = r.outbuf[0:outn] + r.bufStart = 0 + + n, err := r.r.Read(r.inbuf) + r.err = err // save error for when done with buffer + if n > 0 { + r.outbuf = doAppend(&r.rb, r.outbuf, r.inbuf[0:n]) + } + if err == os.EOF { + r.lastBoundary = len(r.outbuf) + } else { + r.lastBoundary = lastBoundary(&r.rb.f, r.outbuf) + if r.lastBoundary == -1 { + r.lastBoundary = 0 + } + } + } + panic("should not reach here") +} + +// Reader returns a new reader that implements Read +// by reading data from r and returning f(data). +func (f Form) Reader(r io.Reader) io.Reader { + const chunk = 4000 + return &normReader{rb: reorderBuffer{f: *formTable[f]}, r: r, inbuf: make([]byte, chunk)} +} diff --git a/src/pkg/exp/norm/readwriter_test.go b/src/pkg/exp/norm/readwriter_test.go new file mode 100644 index 000000000..b415f2b8c --- /dev/null +++ b/src/pkg/exp/norm/readwriter_test.go @@ -0,0 +1,69 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package norm + +import ( + "bytes" + "fmt" + "os" + "strings" + "testing" +) + +var ioTests = []AppendTest{ + {"", strings.Repeat("a\u0316\u0300", 6), strings.Repeat("\u00E0\u0316", 6)}, + {"", strings.Repeat("a\u0300\u0316", 4000), strings.Repeat("\u00E0\u0316", 4000)}, + {"", strings.Repeat("\x80\x80", 4000), strings.Repeat("\x80\x80", 4000)}, + {"", "\u0041\u0307\u0304", "\u01E0"}, +} + +var bufSizes = []int{1, 2, 3, 4, 5, 6, 7, 8, 100, 101, 102, 103, 4000, 4001, 4002, 4003} + +func readFunc(size int) appendFunc { + return func(f Form, out []byte, s string) []byte { + out = append(out, []byte(s)...) + r := f.Reader(bytes.NewBuffer(out)) + buf := make([]byte, size) + result := []byte{} + for n, err := 0, os.Error(nil); err == nil; { + n, err = r.Read(buf) + result = append(result, buf[:n]...) + } + return result + } +} + +func TestReader(t *testing.T) { + for _, s := range bufSizes { + name := fmt.Sprintf("TestReader%da", s) + runAppendTests(t, name, NFKC, readFunc(s), appendTests) + name = fmt.Sprintf("TestReader%db", s) + runAppendTests(t, name, NFKC, readFunc(s), ioTests) + } +} + +func writeFunc(size int) appendFunc { + return func(f Form, out []byte, s string) []byte { + in := append(out, []byte(s)...) + result := new(bytes.Buffer) + w := f.Writer(result) + buf := make([]byte, size) + for n := 0; len(in) > 0; in = in[n:] { + n = copy(buf, in) + _, _ = w.Write(buf[:n]) + } + w.Close() + return result.Bytes() + } +} + +func TestWriter(t *testing.T) { + for _, s := range bufSizes { + name := fmt.Sprintf("TestWriter%da", s) + runAppendTests(t, name, NFKC, writeFunc(s), appendTests) + name = fmt.Sprintf("TestWriter%db", s) + runAppendTests(t, name, NFKC, writeFunc(s), ioTests) + } +} diff --git a/src/pkg/exp/norm/tables.go b/src/pkg/exp/norm/tables.go index 76995c2fa..36dcd5c84 100644 --- a/src/pkg/exp/norm/tables.go +++ b/src/pkg/exp/norm/tables.go @@ -5356,9 +5356,9 @@ var recompMap = map[uint32]uint32{ 0x10A510BA: 0x110AB, } -// charInfoValues: 10944 entries, 21888 bytes +// charInfoValues: 11008 entries, 22016 bytes // Block 2 is the null block. -var charInfoValues = [10944]uint16{ +var charInfoValues = [11008]uint16{ // Block 0x0, offset 0x0 0x003c: 0x8800, 0x003d: 0x8800, 0x003e: 0x8800, // Block 0x1, offset 0x40 @@ -5686,305 +5686,306 @@ var charInfoValues = [10944]uint16{ // Block 0x38, offset 0xe00 0x0e3c: 0x3000, // Block 0x39, offset 0xe40 - 0x0e61: 0x6600, 0x0e62: 0x6600, 0x0e63: 0x6600, - 0x0e64: 0x6600, 0x0e65: 0x6600, 0x0e66: 0x6600, 0x0e67: 0x6600, 0x0e68: 0x6600, 0x0e69: 0x6600, - 0x0e6a: 0x6600, 0x0e6b: 0x6600, 0x0e6c: 0x6600, 0x0e6d: 0x6600, 0x0e6e: 0x6600, 0x0e6f: 0x6600, - 0x0e70: 0x6600, 0x0e71: 0x6600, 0x0e72: 0x6600, 0x0e73: 0x6600, 0x0e74: 0x6600, 0x0e75: 0x6600, + 0x0e40: 0x8800, 0x0e41: 0x8800, 0x0e42: 0x8800, 0x0e43: 0x8800, 0x0e44: 0x8800, 0x0e45: 0x8800, + 0x0e46: 0x8800, 0x0e47: 0x8800, 0x0e48: 0x8800, 0x0e49: 0x8800, 0x0e4a: 0x8800, 0x0e4b: 0x8800, + 0x0e4c: 0x8800, 0x0e4d: 0x8800, 0x0e4e: 0x8800, 0x0e4f: 0x8800, 0x0e50: 0x8800, 0x0e51: 0x8800, + 0x0e52: 0x8800, // Block 0x3a, offset 0xe80 - 0x0ea8: 0x6600, 0x0ea9: 0x6600, - 0x0eaa: 0x6600, 0x0eab: 0x6600, 0x0eac: 0x6600, 0x0ead: 0x6600, 0x0eae: 0x6600, 0x0eaf: 0x6600, - 0x0eb0: 0x6600, 0x0eb1: 0x6600, 0x0eb2: 0x6600, 0x0eb3: 0x6600, 0x0eb4: 0x6600, 0x0eb5: 0x6600, - 0x0eb6: 0x6600, 0x0eb7: 0x6600, 0x0eb8: 0x6600, 0x0eb9: 0x6600, 0x0eba: 0x6600, 0x0ebb: 0x6600, - 0x0ebc: 0x6600, 0x0ebd: 0x6600, 0x0ebe: 0x6600, 0x0ebf: 0x6600, + 0x0ea1: 0xee00, 0x0ea2: 0xee00, 0x0ea3: 0xee00, + 0x0ea4: 0xee00, 0x0ea5: 0xee00, 0x0ea6: 0xee00, 0x0ea7: 0xee00, 0x0ea8: 0xee00, 0x0ea9: 0xee00, + 0x0eaa: 0xee00, 0x0eab: 0xee00, 0x0eac: 0xee00, 0x0ead: 0xee00, 0x0eae: 0xee00, 0x0eaf: 0xee00, + 0x0eb0: 0xee00, 0x0eb1: 0xee00, 0x0eb2: 0xee00, 0x0eb3: 0xee00, 0x0eb4: 0xee00, 0x0eb5: 0xee00, + 0x0eb6: 0xee00, 0x0eb7: 0xee00, 0x0eb8: 0xee00, 0x0eb9: 0xee00, 0x0eba: 0xee00, 0x0ebb: 0xee00, + 0x0ebc: 0xee00, 0x0ebd: 0xee00, 0x0ebe: 0xee00, 0x0ebf: 0xee00, // Block 0x3b, offset 0xec0 - 0x0ec0: 0x6600, 0x0ec1: 0x6600, 0x0ec2: 0x6600, + 0x0ec0: 0xee00, 0x0ec1: 0xee00, 0x0ec2: 0xee00, 0x0ec3: 0xee00, 0x0ec4: 0xee00, 0x0ec5: 0xee00, + 0x0ec6: 0xee00, 0x0ec7: 0xee00, 0x0ec8: 0xee00, 0x0ec9: 0xee00, 0x0eca: 0xee00, 0x0ecb: 0xee00, + 0x0ecc: 0xee00, 0x0ecd: 0xee00, 0x0ece: 0xee00, 0x0ecf: 0xee00, 0x0ed0: 0xee00, 0x0ed1: 0xee00, + 0x0ed2: 0xee00, 0x0ed3: 0xee00, 0x0ed4: 0xee00, 0x0ed5: 0xee00, 0x0ed6: 0xee00, 0x0ed7: 0xee00, + 0x0ed8: 0xee00, 0x0ed9: 0xee00, 0x0eda: 0xee00, 0x0edb: 0xee00, 0x0edc: 0xee00, 0x0edd: 0xee00, + 0x0ede: 0xee00, 0x0edf: 0xee00, 0x0ee0: 0xee00, 0x0ee1: 0xee00, 0x0ee2: 0xee00, 0x0ee3: 0xee00, + 0x0ee4: 0xee00, 0x0ee5: 0xee00, 0x0ee6: 0xee00, 0x0ee7: 0xee00, 0x0ee8: 0xee00, 0x0ee9: 0xee00, + 0x0eea: 0xee00, 0x0eeb: 0xee00, 0x0eec: 0xee00, 0x0eed: 0xee00, 0x0eee: 0xee00, 0x0eef: 0xee00, + 0x0ef0: 0xee00, 0x0ef1: 0xee00, 0x0ef2: 0xee00, 0x0ef3: 0xee00, 0x0ef4: 0xee00, 0x0ef5: 0xee00, + 0x0ef6: 0xee00, 0x0ef7: 0xee00, 0x0ef8: 0xee00, 0x0ef9: 0xee00, 0x0efa: 0xee00, 0x0efb: 0xee00, + 0x0efc: 0xee00, 0x0efd: 0xee00, 0x0efe: 0xee00, 0x0eff: 0xee00, // Block 0x3c, offset 0xf00 - 0x0f1d: 0x00e6, - 0x0f1e: 0x00e6, 0x0f1f: 0x00e6, + 0x0f00: 0xee00, 0x0f01: 0xee00, 0x0f02: 0xee00, // Block 0x3d, offset 0xf40 - 0x0f54: 0x0009, - 0x0f74: 0x0009, + 0x0f5d: 0x00e6, + 0x0f5e: 0x00e6, 0x0f5f: 0x00e6, // Block 0x3e, offset 0xf80 - 0x0f92: 0x0009, - 0x0f9d: 0x00e6, + 0x0f94: 0x0009, + 0x0fb4: 0x0009, // Block 0x3f, offset 0xfc0 - 0x0fe9: 0x00e4, + 0x0fd2: 0x0009, + 0x0fdd: 0x00e6, // Block 0x40, offset 0x1000 - 0x1039: 0x00de, 0x103a: 0x00e6, 0x103b: 0x00dc, + 0x1029: 0x00e4, // Block 0x41, offset 0x1040 - 0x1057: 0x00e6, - 0x1058: 0x00dc, + 0x1079: 0x00de, 0x107a: 0x00e6, 0x107b: 0x00dc, // Block 0x42, offset 0x1080 - 0x10a0: 0x0009, - 0x10b5: 0x00e6, - 0x10b6: 0x00e6, 0x10b7: 0x00e6, 0x10b8: 0x00e6, 0x10b9: 0x00e6, 0x10ba: 0x00e6, 0x10bb: 0x00e6, - 0x10bc: 0x00e6, 0x10bf: 0x00dc, + 0x1097: 0x00e6, + 0x1098: 0x00dc, // Block 0x43, offset 0x10c0 - 0x10c5: 0x8800, - 0x10c6: 0x1100, 0x10c7: 0x8800, 0x10c8: 0x1100, 0x10c9: 0x8800, 0x10ca: 0x1100, 0x10cb: 0x8800, - 0x10cc: 0x1100, 0x10cd: 0x8800, 0x10ce: 0x1100, 0x10d1: 0x8800, - 0x10d2: 0x1100, - 0x10f4: 0x0007, 0x10f5: 0x6600, - 0x10fa: 0x8800, 0x10fb: 0x1100, - 0x10fc: 0x8800, 0x10fd: 0x1100, 0x10fe: 0x8800, 0x10ff: 0x8800, + 0x10e0: 0x0009, + 0x10f5: 0x00e6, + 0x10f6: 0x00e6, 0x10f7: 0x00e6, 0x10f8: 0x00e6, 0x10f9: 0x00e6, 0x10fa: 0x00e6, 0x10fb: 0x00e6, + 0x10fc: 0x00e6, 0x10ff: 0x00dc, // Block 0x44, offset 0x1100 - 0x1100: 0x1100, 0x1101: 0x1100, 0x1102: 0x8800, 0x1103: 0x1100, 0x1104: 0x0009, - 0x112b: 0x00e6, 0x112c: 0x00dc, 0x112d: 0x00e6, 0x112e: 0x00e6, 0x112f: 0x00e6, - 0x1130: 0x00e6, 0x1131: 0x00e6, 0x1132: 0x00e6, 0x1133: 0x00e6, + 0x1105: 0x8800, + 0x1106: 0x1100, 0x1107: 0x8800, 0x1108: 0x1100, 0x1109: 0x8800, 0x110a: 0x1100, 0x110b: 0x8800, + 0x110c: 0x1100, 0x110d: 0x8800, 0x110e: 0x1100, 0x1111: 0x8800, + 0x1112: 0x1100, + 0x1134: 0x0007, 0x1135: 0x6600, + 0x113a: 0x8800, 0x113b: 0x1100, + 0x113c: 0x8800, 0x113d: 0x1100, 0x113e: 0x8800, 0x113f: 0x8800, // Block 0x45, offset 0x1140 - 0x116a: 0x0009, + 0x1140: 0x1100, 0x1141: 0x1100, 0x1142: 0x8800, 0x1143: 0x1100, 0x1144: 0x0009, + 0x116b: 0x00e6, 0x116c: 0x00dc, 0x116d: 0x00e6, 0x116e: 0x00e6, 0x116f: 0x00e6, + 0x1170: 0x00e6, 0x1171: 0x00e6, 0x1172: 0x00e6, 0x1173: 0x00e6, // Block 0x46, offset 0x1180 - 0x11a6: 0x0007, - 0x11b2: 0x0009, 0x11b3: 0x0009, + 0x11aa: 0x0009, // Block 0x47, offset 0x11c0 - 0x11f7: 0x0007, + 0x11e6: 0x0007, + 0x11f2: 0x0009, 0x11f3: 0x0009, // Block 0x48, offset 0x1200 - 0x1210: 0x00e6, 0x1211: 0x00e6, - 0x1212: 0x00e6, 0x1214: 0x0001, 0x1215: 0x00dc, 0x1216: 0x00dc, 0x1217: 0x00dc, - 0x1218: 0x00dc, 0x1219: 0x00dc, 0x121a: 0x00e6, 0x121b: 0x00e6, 0x121c: 0x00dc, 0x121d: 0x00dc, - 0x121e: 0x00dc, 0x121f: 0x00dc, 0x1220: 0x00e6, 0x1222: 0x0001, 0x1223: 0x0001, - 0x1224: 0x0001, 0x1225: 0x0001, 0x1226: 0x0001, 0x1227: 0x0001, 0x1228: 0x0001, - 0x122d: 0x00dc, + 0x1237: 0x0007, // Block 0x49, offset 0x1240 - 0x126c: 0x3000, 0x126d: 0x3000, 0x126e: 0x3000, - 0x1270: 0x3000, 0x1271: 0x3000, 0x1272: 0x3000, 0x1273: 0x3000, 0x1274: 0x3000, 0x1275: 0x3000, - 0x1276: 0x3000, 0x1277: 0x3000, 0x1278: 0x3000, 0x1279: 0x3000, 0x127a: 0x3000, - 0x127c: 0x3000, 0x127d: 0x3000, 0x127e: 0x3000, 0x127f: 0x3000, + 0x1250: 0x00e6, 0x1251: 0x00e6, + 0x1252: 0x00e6, 0x1254: 0x0001, 0x1255: 0x00dc, 0x1256: 0x00dc, 0x1257: 0x00dc, + 0x1258: 0x00dc, 0x1259: 0x00dc, 0x125a: 0x00e6, 0x125b: 0x00e6, 0x125c: 0x00dc, 0x125d: 0x00dc, + 0x125e: 0x00dc, 0x125f: 0x00dc, 0x1260: 0x00e6, 0x1262: 0x0001, 0x1263: 0x0001, + 0x1264: 0x0001, 0x1265: 0x0001, 0x1266: 0x0001, 0x1267: 0x0001, 0x1268: 0x0001, + 0x126d: 0x00dc, // Block 0x4a, offset 0x1280 - 0x1280: 0x3000, 0x1281: 0x3000, 0x1282: 0x3000, 0x1283: 0x3000, 0x1284: 0x3000, 0x1285: 0x3000, - 0x1286: 0x3000, 0x1287: 0x3000, 0x1288: 0x3000, 0x1289: 0x3000, 0x128a: 0x3000, 0x128b: 0x3000, - 0x128c: 0x3000, 0x128d: 0x3000, 0x128f: 0x3000, 0x1290: 0x3000, 0x1291: 0x3000, - 0x1292: 0x3000, 0x1293: 0x3000, 0x1294: 0x3000, 0x1295: 0x3000, 0x1296: 0x3000, 0x1297: 0x3000, - 0x1298: 0x3000, 0x1299: 0x3000, 0x129a: 0x3000, 0x129b: 0x3000, 0x129c: 0x3000, 0x129d: 0x3000, - 0x129e: 0x3000, 0x129f: 0x3000, 0x12a0: 0x3000, 0x12a1: 0x3000, 0x12a2: 0x3000, 0x12a3: 0x3000, - 0x12a4: 0x3000, 0x12a5: 0x3000, 0x12a6: 0x3000, 0x12a7: 0x3000, 0x12a8: 0x3000, 0x12a9: 0x3000, - 0x12aa: 0x3000, - 0x12b8: 0x3000, + 0x12ac: 0x3000, 0x12ad: 0x3000, 0x12ae: 0x3000, + 0x12b0: 0x3000, 0x12b1: 0x3000, 0x12b2: 0x3000, 0x12b3: 0x3000, 0x12b4: 0x3000, 0x12b5: 0x3000, + 0x12b6: 0x3000, 0x12b7: 0x3000, 0x12b8: 0x3000, 0x12b9: 0x3000, 0x12ba: 0x3000, + 0x12bc: 0x3000, 0x12bd: 0x3000, 0x12be: 0x3000, 0x12bf: 0x3000, // Block 0x4b, offset 0x12c0 - 0x12db: 0x3000, 0x12dc: 0x3000, 0x12dd: 0x3000, + 0x12c0: 0x3000, 0x12c1: 0x3000, 0x12c2: 0x3000, 0x12c3: 0x3000, 0x12c4: 0x3000, 0x12c5: 0x3000, + 0x12c6: 0x3000, 0x12c7: 0x3000, 0x12c8: 0x3000, 0x12c9: 0x3000, 0x12ca: 0x3000, 0x12cb: 0x3000, + 0x12cc: 0x3000, 0x12cd: 0x3000, 0x12cf: 0x3000, 0x12d0: 0x3000, 0x12d1: 0x3000, + 0x12d2: 0x3000, 0x12d3: 0x3000, 0x12d4: 0x3000, 0x12d5: 0x3000, 0x12d6: 0x3000, 0x12d7: 0x3000, + 0x12d8: 0x3000, 0x12d9: 0x3000, 0x12da: 0x3000, 0x12db: 0x3000, 0x12dc: 0x3000, 0x12dd: 0x3000, 0x12de: 0x3000, 0x12df: 0x3000, 0x12e0: 0x3000, 0x12e1: 0x3000, 0x12e2: 0x3000, 0x12e3: 0x3000, 0x12e4: 0x3000, 0x12e5: 0x3000, 0x12e6: 0x3000, 0x12e7: 0x3000, 0x12e8: 0x3000, 0x12e9: 0x3000, - 0x12ea: 0x3000, 0x12eb: 0x3000, 0x12ec: 0x3000, 0x12ed: 0x3000, 0x12ee: 0x3000, 0x12ef: 0x3000, - 0x12f0: 0x3000, 0x12f1: 0x3000, 0x12f2: 0x3000, 0x12f3: 0x3000, 0x12f4: 0x3000, 0x12f5: 0x3000, - 0x12f6: 0x3000, 0x12f7: 0x3000, 0x12f8: 0x3000, 0x12f9: 0x3000, 0x12fa: 0x3000, 0x12fb: 0x3000, - 0x12fc: 0x3000, 0x12fd: 0x3000, 0x12fe: 0x3000, 0x12ff: 0x3000, + 0x12ea: 0x3000, + 0x12f8: 0x3000, // Block 0x4c, offset 0x1300 - 0x1300: 0x00e6, 0x1301: 0x00e6, 0x1302: 0x00dc, 0x1303: 0x00e6, 0x1304: 0x00e6, 0x1305: 0x00e6, - 0x1306: 0x00e6, 0x1307: 0x00e6, 0x1308: 0x00e6, 0x1309: 0x00e6, 0x130a: 0x00dc, 0x130b: 0x00e6, - 0x130c: 0x00e6, 0x130d: 0x00ea, 0x130e: 0x00d6, 0x130f: 0x00dc, 0x1310: 0x00ca, 0x1311: 0x00e6, - 0x1312: 0x00e6, 0x1313: 0x00e6, 0x1314: 0x00e6, 0x1315: 0x00e6, 0x1316: 0x00e6, 0x1317: 0x00e6, - 0x1318: 0x00e6, 0x1319: 0x00e6, 0x131a: 0x00e6, 0x131b: 0x00e6, 0x131c: 0x00e6, 0x131d: 0x00e6, - 0x131e: 0x00e6, 0x131f: 0x00e6, 0x1320: 0x00e6, 0x1321: 0x00e6, 0x1322: 0x00e6, 0x1323: 0x00e6, - 0x1324: 0x00e6, 0x1325: 0x00e6, 0x1326: 0x00e6, - 0x133c: 0x00e9, 0x133d: 0x00dc, 0x133e: 0x00e6, 0x133f: 0x00dc, + 0x131b: 0x3000, 0x131c: 0x3000, 0x131d: 0x3000, + 0x131e: 0x3000, 0x131f: 0x3000, 0x1320: 0x3000, 0x1321: 0x3000, 0x1322: 0x3000, 0x1323: 0x3000, + 0x1324: 0x3000, 0x1325: 0x3000, 0x1326: 0x3000, 0x1327: 0x3000, 0x1328: 0x3000, 0x1329: 0x3000, + 0x132a: 0x3000, 0x132b: 0x3000, 0x132c: 0x3000, 0x132d: 0x3000, 0x132e: 0x3000, 0x132f: 0x3000, + 0x1330: 0x3000, 0x1331: 0x3000, 0x1332: 0x3000, 0x1333: 0x3000, 0x1334: 0x3000, 0x1335: 0x3000, + 0x1336: 0x3000, 0x1337: 0x3000, 0x1338: 0x3000, 0x1339: 0x3000, 0x133a: 0x3000, 0x133b: 0x3000, + 0x133c: 0x3000, 0x133d: 0x3000, 0x133e: 0x3000, 0x133f: 0x3000, // Block 0x4d, offset 0x1340 - 0x1340: 0x1100, 0x1341: 0x1100, 0x1342: 0x1100, 0x1343: 0x1100, 0x1344: 0x1100, 0x1345: 0x1100, - 0x1346: 0x1100, 0x1347: 0x1100, 0x1348: 0x1100, 0x1349: 0x1100, 0x134a: 0x1100, 0x134b: 0x1100, - 0x134c: 0x1100, 0x134d: 0x1100, 0x134e: 0x1100, 0x134f: 0x1100, 0x1350: 0x1100, 0x1351: 0x1100, - 0x1352: 0x1100, 0x1353: 0x1100, 0x1354: 0x1100, 0x1355: 0x1100, 0x1356: 0x1100, 0x1357: 0x1100, - 0x1358: 0x1100, 0x1359: 0x1100, 0x135a: 0x1100, 0x135b: 0x1100, 0x135c: 0x1100, 0x135d: 0x1100, - 0x135e: 0x1100, 0x135f: 0x1100, 0x1360: 0x1100, 0x1361: 0x1100, 0x1362: 0x1100, 0x1363: 0x1100, - 0x1364: 0x1100, 0x1365: 0x1100, 0x1366: 0x1100, 0x1367: 0x1100, 0x1368: 0x1100, 0x1369: 0x1100, - 0x136a: 0x1100, 0x136b: 0x1100, 0x136c: 0x1100, 0x136d: 0x1100, 0x136e: 0x1100, 0x136f: 0x1100, - 0x1370: 0x1100, 0x1371: 0x1100, 0x1372: 0x1100, 0x1373: 0x1100, 0x1374: 0x1100, 0x1375: 0x1100, - 0x1376: 0x9900, 0x1377: 0x9900, 0x1378: 0x1100, 0x1379: 0x1100, 0x137a: 0x1100, 0x137b: 0x1100, - 0x137c: 0x1100, 0x137d: 0x1100, 0x137e: 0x1100, 0x137f: 0x1100, + 0x1340: 0x00e6, 0x1341: 0x00e6, 0x1342: 0x00dc, 0x1343: 0x00e6, 0x1344: 0x00e6, 0x1345: 0x00e6, + 0x1346: 0x00e6, 0x1347: 0x00e6, 0x1348: 0x00e6, 0x1349: 0x00e6, 0x134a: 0x00dc, 0x134b: 0x00e6, + 0x134c: 0x00e6, 0x134d: 0x00ea, 0x134e: 0x00d6, 0x134f: 0x00dc, 0x1350: 0x00ca, 0x1351: 0x00e6, + 0x1352: 0x00e6, 0x1353: 0x00e6, 0x1354: 0x00e6, 0x1355: 0x00e6, 0x1356: 0x00e6, 0x1357: 0x00e6, + 0x1358: 0x00e6, 0x1359: 0x00e6, 0x135a: 0x00e6, 0x135b: 0x00e6, 0x135c: 0x00e6, 0x135d: 0x00e6, + 0x135e: 0x00e6, 0x135f: 0x00e6, 0x1360: 0x00e6, 0x1361: 0x00e6, 0x1362: 0x00e6, 0x1363: 0x00e6, + 0x1364: 0x00e6, 0x1365: 0x00e6, 0x1366: 0x00e6, + 0x137c: 0x00e9, 0x137d: 0x00dc, 0x137e: 0x00e6, 0x137f: 0x00dc, // Block 0x4e, offset 0x1380 0x1380: 0x1100, 0x1381: 0x1100, 0x1382: 0x1100, 0x1383: 0x1100, 0x1384: 0x1100, 0x1385: 0x1100, 0x1386: 0x1100, 0x1387: 0x1100, 0x1388: 0x1100, 0x1389: 0x1100, 0x138a: 0x1100, 0x138b: 0x1100, 0x138c: 0x1100, 0x138d: 0x1100, 0x138e: 0x1100, 0x138f: 0x1100, 0x1390: 0x1100, 0x1391: 0x1100, 0x1392: 0x1100, 0x1393: 0x1100, 0x1394: 0x1100, 0x1395: 0x1100, 0x1396: 0x1100, 0x1397: 0x1100, - 0x1398: 0x1100, 0x1399: 0x1100, 0x139a: 0x9900, 0x139b: 0x9900, 0x139c: 0x1100, 0x139d: 0x1100, - 0x139e: 0x1100, 0x139f: 0x1100, 0x13a0: 0x1100, 0x13a1: 0x1100, 0x13a2: 0x9900, 0x13a3: 0x9900, + 0x1398: 0x1100, 0x1399: 0x1100, 0x139a: 0x1100, 0x139b: 0x1100, 0x139c: 0x1100, 0x139d: 0x1100, + 0x139e: 0x1100, 0x139f: 0x1100, 0x13a0: 0x1100, 0x13a1: 0x1100, 0x13a2: 0x1100, 0x13a3: 0x1100, 0x13a4: 0x1100, 0x13a5: 0x1100, 0x13a6: 0x1100, 0x13a7: 0x1100, 0x13a8: 0x1100, 0x13a9: 0x1100, 0x13aa: 0x1100, 0x13ab: 0x1100, 0x13ac: 0x1100, 0x13ad: 0x1100, 0x13ae: 0x1100, 0x13af: 0x1100, 0x13b0: 0x1100, 0x13b1: 0x1100, 0x13b2: 0x1100, 0x13b3: 0x1100, 0x13b4: 0x1100, 0x13b5: 0x1100, - 0x13b6: 0x1100, 0x13b7: 0x1100, 0x13b8: 0x1100, 0x13b9: 0x1100, 0x13ba: 0x1100, 0x13bb: 0x1100, + 0x13b6: 0x9900, 0x13b7: 0x9900, 0x13b8: 0x1100, 0x13b9: 0x1100, 0x13ba: 0x1100, 0x13bb: 0x1100, 0x13bc: 0x1100, 0x13bd: 0x1100, 0x13be: 0x1100, 0x13bf: 0x1100, // Block 0x4f, offset 0x13c0 0x13c0: 0x1100, 0x13c1: 0x1100, 0x13c2: 0x1100, 0x13c3: 0x1100, 0x13c4: 0x1100, 0x13c5: 0x1100, 0x13c6: 0x1100, 0x13c7: 0x1100, 0x13c8: 0x1100, 0x13c9: 0x1100, 0x13ca: 0x1100, 0x13cb: 0x1100, 0x13cc: 0x1100, 0x13cd: 0x1100, 0x13ce: 0x1100, 0x13cf: 0x1100, 0x13d0: 0x1100, 0x13d1: 0x1100, 0x13d2: 0x1100, 0x13d3: 0x1100, 0x13d4: 0x1100, 0x13d5: 0x1100, 0x13d6: 0x1100, 0x13d7: 0x1100, - 0x13d8: 0x1100, 0x13d9: 0x1100, 0x13da: 0x3000, 0x13db: 0x3100, - 0x13e0: 0x9900, 0x13e1: 0x9900, 0x13e2: 0x1100, 0x13e3: 0x1100, + 0x13d8: 0x1100, 0x13d9: 0x1100, 0x13da: 0x9900, 0x13db: 0x9900, 0x13dc: 0x1100, 0x13dd: 0x1100, + 0x13de: 0x1100, 0x13df: 0x1100, 0x13e0: 0x1100, 0x13e1: 0x1100, 0x13e2: 0x9900, 0x13e3: 0x9900, 0x13e4: 0x1100, 0x13e5: 0x1100, 0x13e6: 0x1100, 0x13e7: 0x1100, 0x13e8: 0x1100, 0x13e9: 0x1100, 0x13ea: 0x1100, 0x13eb: 0x1100, 0x13ec: 0x1100, 0x13ed: 0x1100, 0x13ee: 0x1100, 0x13ef: 0x1100, 0x13f0: 0x1100, 0x13f1: 0x1100, 0x13f2: 0x1100, 0x13f3: 0x1100, 0x13f4: 0x1100, 0x13f5: 0x1100, - 0x13f6: 0x1100, 0x13f7: 0x1100, 0x13f8: 0x9900, 0x13f9: 0x9900, 0x13fa: 0x1100, 0x13fb: 0x1100, + 0x13f6: 0x1100, 0x13f7: 0x1100, 0x13f8: 0x1100, 0x13f9: 0x1100, 0x13fa: 0x1100, 0x13fb: 0x1100, 0x13fc: 0x1100, 0x13fd: 0x1100, 0x13fe: 0x1100, 0x13ff: 0x1100, // Block 0x50, offset 0x1400 0x1400: 0x1100, 0x1401: 0x1100, 0x1402: 0x1100, 0x1403: 0x1100, 0x1404: 0x1100, 0x1405: 0x1100, 0x1406: 0x1100, 0x1407: 0x1100, 0x1408: 0x1100, 0x1409: 0x1100, 0x140a: 0x1100, 0x140b: 0x1100, - 0x140c: 0x9900, 0x140d: 0x9900, 0x140e: 0x1100, 0x140f: 0x1100, 0x1410: 0x1100, 0x1411: 0x1100, + 0x140c: 0x1100, 0x140d: 0x1100, 0x140e: 0x1100, 0x140f: 0x1100, 0x1410: 0x1100, 0x1411: 0x1100, 0x1412: 0x1100, 0x1413: 0x1100, 0x1414: 0x1100, 0x1415: 0x1100, 0x1416: 0x1100, 0x1417: 0x1100, - 0x1418: 0x1100, 0x1419: 0x1100, 0x141a: 0x1100, 0x141b: 0x1100, 0x141c: 0x1100, 0x141d: 0x1100, - 0x141e: 0x1100, 0x141f: 0x1100, 0x1420: 0x1100, 0x1421: 0x1100, 0x1422: 0x1100, 0x1423: 0x1100, + 0x1418: 0x1100, 0x1419: 0x1100, 0x141a: 0x3000, 0x141b: 0x3100, + 0x1420: 0x9900, 0x1421: 0x9900, 0x1422: 0x1100, 0x1423: 0x1100, 0x1424: 0x1100, 0x1425: 0x1100, 0x1426: 0x1100, 0x1427: 0x1100, 0x1428: 0x1100, 0x1429: 0x1100, 0x142a: 0x1100, 0x142b: 0x1100, 0x142c: 0x1100, 0x142d: 0x1100, 0x142e: 0x1100, 0x142f: 0x1100, 0x1430: 0x1100, 0x1431: 0x1100, 0x1432: 0x1100, 0x1433: 0x1100, 0x1434: 0x1100, 0x1435: 0x1100, - 0x1436: 0x1100, 0x1437: 0x1100, 0x1438: 0x1100, 0x1439: 0x1100, + 0x1436: 0x1100, 0x1437: 0x1100, 0x1438: 0x9900, 0x1439: 0x9900, 0x143a: 0x1100, 0x143b: 0x1100, + 0x143c: 0x1100, 0x143d: 0x1100, 0x143e: 0x1100, 0x143f: 0x1100, // Block 0x51, offset 0x1440 - 0x1440: 0x9900, 0x1441: 0x9900, 0x1442: 0x9900, 0x1443: 0x9900, 0x1444: 0x9900, 0x1445: 0x9900, - 0x1446: 0x9900, 0x1447: 0x9900, 0x1448: 0x9900, 0x1449: 0x9900, 0x144a: 0x9900, 0x144b: 0x9900, - 0x144c: 0x9900, 0x144d: 0x9900, 0x144e: 0x9900, 0x144f: 0x9900, 0x1450: 0x9900, 0x1451: 0x9900, - 0x1452: 0x1100, 0x1453: 0x1100, 0x1454: 0x1100, 0x1455: 0x1100, - 0x1458: 0x9900, 0x1459: 0x9900, 0x145a: 0x1100, 0x145b: 0x1100, 0x145c: 0x1100, 0x145d: 0x1100, - 0x1460: 0x9900, 0x1461: 0x9900, 0x1462: 0x9900, 0x1463: 0x9900, - 0x1464: 0x9900, 0x1465: 0x9900, 0x1466: 0x9900, 0x1467: 0x9900, 0x1468: 0x9900, 0x1469: 0x9900, - 0x146a: 0x9900, 0x146b: 0x9900, 0x146c: 0x9900, 0x146d: 0x9900, 0x146e: 0x9900, 0x146f: 0x9900, - 0x1470: 0x9900, 0x1471: 0x9900, 0x1472: 0x1100, 0x1473: 0x1100, 0x1474: 0x1100, 0x1475: 0x1100, - 0x1476: 0x1100, 0x1477: 0x1100, 0x1478: 0x9900, 0x1479: 0x9900, 0x147a: 0x1100, 0x147b: 0x1100, - 0x147c: 0x1100, 0x147d: 0x1100, 0x147e: 0x1100, 0x147f: 0x1100, + 0x1440: 0x1100, 0x1441: 0x1100, 0x1442: 0x1100, 0x1443: 0x1100, 0x1444: 0x1100, 0x1445: 0x1100, + 0x1446: 0x1100, 0x1447: 0x1100, 0x1448: 0x1100, 0x1449: 0x1100, 0x144a: 0x1100, 0x144b: 0x1100, + 0x144c: 0x9900, 0x144d: 0x9900, 0x144e: 0x1100, 0x144f: 0x1100, 0x1450: 0x1100, 0x1451: 0x1100, + 0x1452: 0x1100, 0x1453: 0x1100, 0x1454: 0x1100, 0x1455: 0x1100, 0x1456: 0x1100, 0x1457: 0x1100, + 0x1458: 0x1100, 0x1459: 0x1100, 0x145a: 0x1100, 0x145b: 0x1100, 0x145c: 0x1100, 0x145d: 0x1100, + 0x145e: 0x1100, 0x145f: 0x1100, 0x1460: 0x1100, 0x1461: 0x1100, 0x1462: 0x1100, 0x1463: 0x1100, + 0x1464: 0x1100, 0x1465: 0x1100, 0x1466: 0x1100, 0x1467: 0x1100, 0x1468: 0x1100, 0x1469: 0x1100, + 0x146a: 0x1100, 0x146b: 0x1100, 0x146c: 0x1100, 0x146d: 0x1100, 0x146e: 0x1100, 0x146f: 0x1100, + 0x1470: 0x1100, 0x1471: 0x1100, 0x1472: 0x1100, 0x1473: 0x1100, 0x1474: 0x1100, 0x1475: 0x1100, + 0x1476: 0x1100, 0x1477: 0x1100, 0x1478: 0x1100, 0x1479: 0x1100, // Block 0x52, offset 0x1480 - 0x1480: 0x9900, 0x1481: 0x9900, 0x1482: 0x1100, 0x1483: 0x1100, 0x1484: 0x1100, 0x1485: 0x1100, - 0x1488: 0x9900, 0x1489: 0x9900, 0x148a: 0x1100, 0x148b: 0x1100, - 0x148c: 0x1100, 0x148d: 0x1100, 0x1490: 0x9900, 0x1491: 0x9900, - 0x1492: 0x1100, 0x1493: 0x1100, 0x1494: 0x1100, 0x1495: 0x1100, 0x1496: 0x1100, 0x1497: 0x1100, - 0x1499: 0x9900, 0x149b: 0x1100, 0x149d: 0x1100, - 0x149f: 0x1100, 0x14a0: 0x9900, 0x14a1: 0x9900, 0x14a2: 0x9900, 0x14a3: 0x9900, + 0x1480: 0x9900, 0x1481: 0x9900, 0x1482: 0x9900, 0x1483: 0x9900, 0x1484: 0x9900, 0x1485: 0x9900, + 0x1486: 0x9900, 0x1487: 0x9900, 0x1488: 0x9900, 0x1489: 0x9900, 0x148a: 0x9900, 0x148b: 0x9900, + 0x148c: 0x9900, 0x148d: 0x9900, 0x148e: 0x9900, 0x148f: 0x9900, 0x1490: 0x9900, 0x1491: 0x9900, + 0x1492: 0x1100, 0x1493: 0x1100, 0x1494: 0x1100, 0x1495: 0x1100, + 0x1498: 0x9900, 0x1499: 0x9900, 0x149a: 0x1100, 0x149b: 0x1100, 0x149c: 0x1100, 0x149d: 0x1100, + 0x14a0: 0x9900, 0x14a1: 0x9900, 0x14a2: 0x9900, 0x14a3: 0x9900, 0x14a4: 0x9900, 0x14a5: 0x9900, 0x14a6: 0x9900, 0x14a7: 0x9900, 0x14a8: 0x9900, 0x14a9: 0x9900, 0x14aa: 0x9900, 0x14ab: 0x9900, 0x14ac: 0x9900, 0x14ad: 0x9900, 0x14ae: 0x9900, 0x14af: 0x9900, - 0x14b0: 0x9900, 0x14b1: 0x3300, 0x14b2: 0x1100, 0x14b3: 0x3300, 0x14b4: 0x9900, 0x14b5: 0x3300, - 0x14b6: 0x1100, 0x14b7: 0x3300, 0x14b8: 0x1100, 0x14b9: 0x3300, 0x14ba: 0x1100, 0x14bb: 0x3300, - 0x14bc: 0x9900, 0x14bd: 0x3300, + 0x14b0: 0x9900, 0x14b1: 0x9900, 0x14b2: 0x1100, 0x14b3: 0x1100, 0x14b4: 0x1100, 0x14b5: 0x1100, + 0x14b6: 0x1100, 0x14b7: 0x1100, 0x14b8: 0x9900, 0x14b9: 0x9900, 0x14ba: 0x1100, 0x14bb: 0x1100, + 0x14bc: 0x1100, 0x14bd: 0x1100, 0x14be: 0x1100, 0x14bf: 0x1100, // Block 0x53, offset 0x14c0 - 0x14c0: 0x1100, 0x14c1: 0x1100, 0x14c2: 0x1100, 0x14c3: 0x1100, 0x14c4: 0x1100, 0x14c5: 0x1100, - 0x14c6: 0x1100, 0x14c7: 0x1100, 0x14c8: 0x1100, 0x14c9: 0x1100, 0x14ca: 0x1100, 0x14cb: 0x1100, - 0x14cc: 0x1100, 0x14cd: 0x1100, 0x14ce: 0x1100, 0x14cf: 0x1100, 0x14d0: 0x1100, 0x14d1: 0x1100, + 0x14c0: 0x9900, 0x14c1: 0x9900, 0x14c2: 0x1100, 0x14c3: 0x1100, 0x14c4: 0x1100, 0x14c5: 0x1100, + 0x14c8: 0x9900, 0x14c9: 0x9900, 0x14ca: 0x1100, 0x14cb: 0x1100, + 0x14cc: 0x1100, 0x14cd: 0x1100, 0x14d0: 0x9900, 0x14d1: 0x9900, 0x14d2: 0x1100, 0x14d3: 0x1100, 0x14d4: 0x1100, 0x14d5: 0x1100, 0x14d6: 0x1100, 0x14d7: 0x1100, - 0x14d8: 0x1100, 0x14d9: 0x1100, 0x14da: 0x1100, 0x14db: 0x1100, 0x14dc: 0x1100, 0x14dd: 0x1100, - 0x14de: 0x1100, 0x14df: 0x1100, 0x14e0: 0x1100, 0x14e1: 0x1100, 0x14e2: 0x1100, 0x14e3: 0x1100, - 0x14e4: 0x1100, 0x14e5: 0x1100, 0x14e6: 0x1100, 0x14e7: 0x1100, 0x14e8: 0x1100, 0x14e9: 0x1100, - 0x14ea: 0x1100, 0x14eb: 0x1100, 0x14ec: 0x1100, 0x14ed: 0x1100, 0x14ee: 0x1100, 0x14ef: 0x1100, - 0x14f0: 0x1100, 0x14f1: 0x1100, 0x14f2: 0x1100, 0x14f3: 0x1100, 0x14f4: 0x1100, - 0x14f6: 0x9900, 0x14f7: 0x1100, 0x14f8: 0x1100, 0x14f9: 0x1100, 0x14fa: 0x1100, 0x14fb: 0x3300, - 0x14fc: 0x1100, 0x14fd: 0x3000, 0x14fe: 0x3300, 0x14ff: 0x3800, + 0x14d9: 0x9900, 0x14db: 0x1100, 0x14dd: 0x1100, + 0x14df: 0x1100, 0x14e0: 0x9900, 0x14e1: 0x9900, 0x14e2: 0x9900, 0x14e3: 0x9900, + 0x14e4: 0x9900, 0x14e5: 0x9900, 0x14e6: 0x9900, 0x14e7: 0x9900, 0x14e8: 0x9900, 0x14e9: 0x9900, + 0x14ea: 0x9900, 0x14eb: 0x9900, 0x14ec: 0x9900, 0x14ed: 0x9900, 0x14ee: 0x9900, 0x14ef: 0x9900, + 0x14f0: 0x9900, 0x14f1: 0x3300, 0x14f2: 0x1100, 0x14f3: 0x3300, 0x14f4: 0x9900, 0x14f5: 0x3300, + 0x14f6: 0x1100, 0x14f7: 0x3300, 0x14f8: 0x1100, 0x14f9: 0x3300, 0x14fa: 0x1100, 0x14fb: 0x3300, + 0x14fc: 0x9900, 0x14fd: 0x3300, // Block 0x54, offset 0x1500 - 0x1500: 0x3000, 0x1501: 0x3100, 0x1502: 0x1100, 0x1503: 0x1100, 0x1504: 0x1100, - 0x1506: 0x9900, 0x1507: 0x1100, 0x1508: 0x1100, 0x1509: 0x3300, 0x150a: 0x1100, 0x150b: 0x3300, - 0x150c: 0x1100, 0x150d: 0x3100, 0x150e: 0x3100, 0x150f: 0x3100, 0x1510: 0x1100, 0x1511: 0x1100, - 0x1512: 0x1100, 0x1513: 0x3300, 0x1516: 0x1100, 0x1517: 0x1100, - 0x1518: 0x1100, 0x1519: 0x1100, 0x151a: 0x1100, 0x151b: 0x3300, 0x151d: 0x3100, - 0x151e: 0x3100, 0x151f: 0x3100, 0x1520: 0x1100, 0x1521: 0x1100, 0x1522: 0x1100, 0x1523: 0x3300, + 0x1500: 0x1100, 0x1501: 0x1100, 0x1502: 0x1100, 0x1503: 0x1100, 0x1504: 0x1100, 0x1505: 0x1100, + 0x1506: 0x1100, 0x1507: 0x1100, 0x1508: 0x1100, 0x1509: 0x1100, 0x150a: 0x1100, 0x150b: 0x1100, + 0x150c: 0x1100, 0x150d: 0x1100, 0x150e: 0x1100, 0x150f: 0x1100, 0x1510: 0x1100, 0x1511: 0x1100, + 0x1512: 0x1100, 0x1513: 0x1100, 0x1514: 0x1100, 0x1515: 0x1100, 0x1516: 0x1100, 0x1517: 0x1100, + 0x1518: 0x1100, 0x1519: 0x1100, 0x151a: 0x1100, 0x151b: 0x1100, 0x151c: 0x1100, 0x151d: 0x1100, + 0x151e: 0x1100, 0x151f: 0x1100, 0x1520: 0x1100, 0x1521: 0x1100, 0x1522: 0x1100, 0x1523: 0x1100, 0x1524: 0x1100, 0x1525: 0x1100, 0x1526: 0x1100, 0x1527: 0x1100, 0x1528: 0x1100, 0x1529: 0x1100, - 0x152a: 0x1100, 0x152b: 0x3300, 0x152c: 0x1100, 0x152d: 0x3100, 0x152e: 0x3300, 0x152f: 0x3300, - 0x1532: 0x1100, 0x1533: 0x1100, 0x1534: 0x1100, - 0x1536: 0x9900, 0x1537: 0x1100, 0x1538: 0x1100, 0x1539: 0x3300, 0x153a: 0x1100, 0x153b: 0x3300, - 0x153c: 0x1100, 0x153d: 0x3300, 0x153e: 0x3800, + 0x152a: 0x1100, 0x152b: 0x1100, 0x152c: 0x1100, 0x152d: 0x1100, 0x152e: 0x1100, 0x152f: 0x1100, + 0x1530: 0x1100, 0x1531: 0x1100, 0x1532: 0x1100, 0x1533: 0x1100, 0x1534: 0x1100, + 0x1536: 0x9900, 0x1537: 0x1100, 0x1538: 0x1100, 0x1539: 0x1100, 0x153a: 0x1100, 0x153b: 0x3300, + 0x153c: 0x1100, 0x153d: 0x3000, 0x153e: 0x3300, 0x153f: 0x3800, // Block 0x55, offset 0x1540 - 0x1540: 0x3300, 0x1541: 0x3300, 0x1542: 0x3000, 0x1543: 0x3000, 0x1544: 0x3000, 0x1545: 0x3000, - 0x1546: 0x3000, 0x1547: 0x3000, 0x1548: 0x3000, 0x1549: 0x3000, 0x154a: 0x3000, - 0x1551: 0x3000, - 0x1557: 0x3000, - 0x1564: 0x3000, 0x1565: 0x3000, 0x1566: 0x3000, - 0x156f: 0x3000, - 0x1573: 0x3000, 0x1574: 0x3000, - 0x1576: 0x3000, 0x1577: 0x3000, - 0x157c: 0x3000, 0x157e: 0x3000, + 0x1540: 0x3000, 0x1541: 0x3100, 0x1542: 0x1100, 0x1543: 0x1100, 0x1544: 0x1100, + 0x1546: 0x9900, 0x1547: 0x1100, 0x1548: 0x1100, 0x1549: 0x3300, 0x154a: 0x1100, 0x154b: 0x3300, + 0x154c: 0x1100, 0x154d: 0x3100, 0x154e: 0x3100, 0x154f: 0x3100, 0x1550: 0x1100, 0x1551: 0x1100, + 0x1552: 0x1100, 0x1553: 0x3300, 0x1556: 0x1100, 0x1557: 0x1100, + 0x1558: 0x1100, 0x1559: 0x1100, 0x155a: 0x1100, 0x155b: 0x3300, 0x155d: 0x3100, + 0x155e: 0x3100, 0x155f: 0x3100, 0x1560: 0x1100, 0x1561: 0x1100, 0x1562: 0x1100, 0x1563: 0x3300, + 0x1564: 0x1100, 0x1565: 0x1100, 0x1566: 0x1100, 0x1567: 0x1100, 0x1568: 0x1100, 0x1569: 0x1100, + 0x156a: 0x1100, 0x156b: 0x3300, 0x156c: 0x1100, 0x156d: 0x3100, 0x156e: 0x3300, 0x156f: 0x3300, + 0x1572: 0x1100, 0x1573: 0x1100, 0x1574: 0x1100, + 0x1576: 0x9900, 0x1577: 0x1100, 0x1578: 0x1100, 0x1579: 0x3300, 0x157a: 0x1100, 0x157b: 0x3300, + 0x157c: 0x1100, 0x157d: 0x3300, 0x157e: 0x3800, // Block 0x56, offset 0x1580 - 0x1587: 0x3000, 0x1588: 0x3000, 0x1589: 0x3000, + 0x1580: 0x3300, 0x1581: 0x3300, 0x1582: 0x3000, 0x1583: 0x3000, 0x1584: 0x3000, 0x1585: 0x3000, + 0x1586: 0x3000, 0x1587: 0x3000, 0x1588: 0x3000, 0x1589: 0x3000, 0x158a: 0x3000, + 0x1591: 0x3000, 0x1597: 0x3000, - 0x159f: 0x3000, - 0x15b0: 0x3000, 0x15b1: 0x3000, 0x15b4: 0x3000, 0x15b5: 0x3000, - 0x15b6: 0x3000, 0x15b7: 0x3000, 0x15b8: 0x3000, 0x15b9: 0x3000, 0x15ba: 0x3000, 0x15bb: 0x3000, - 0x15bc: 0x3000, 0x15bd: 0x3000, 0x15be: 0x3000, 0x15bf: 0x3000, + 0x15a4: 0x3000, 0x15a5: 0x3000, 0x15a6: 0x3000, + 0x15af: 0x3000, + 0x15b3: 0x3000, 0x15b4: 0x3000, + 0x15b6: 0x3000, 0x15b7: 0x3000, + 0x15bc: 0x3000, 0x15be: 0x3000, // Block 0x57, offset 0x15c0 - 0x15c0: 0x3000, 0x15c1: 0x3000, 0x15c2: 0x3000, 0x15c3: 0x3000, 0x15c4: 0x3000, 0x15c5: 0x3000, - 0x15c6: 0x3000, 0x15c7: 0x3000, 0x15c8: 0x3000, 0x15c9: 0x3000, 0x15ca: 0x3000, 0x15cb: 0x3000, - 0x15cc: 0x3000, 0x15cd: 0x3000, 0x15ce: 0x3000, 0x15d0: 0x3000, 0x15d1: 0x3000, - 0x15d2: 0x3000, 0x15d3: 0x3000, 0x15d4: 0x3000, 0x15d5: 0x3000, 0x15d6: 0x3000, 0x15d7: 0x3000, - 0x15d8: 0x3000, 0x15d9: 0x3000, 0x15da: 0x3000, 0x15db: 0x3000, 0x15dc: 0x3000, - 0x15e8: 0x3000, + 0x15c7: 0x3000, 0x15c8: 0x3000, 0x15c9: 0x3000, + 0x15d7: 0x3000, + 0x15df: 0x3000, + 0x15f0: 0x3000, 0x15f1: 0x3000, 0x15f4: 0x3000, 0x15f5: 0x3000, + 0x15f6: 0x3000, 0x15f7: 0x3000, 0x15f8: 0x3000, 0x15f9: 0x3000, 0x15fa: 0x3000, 0x15fb: 0x3000, + 0x15fc: 0x3000, 0x15fd: 0x3000, 0x15fe: 0x3000, 0x15ff: 0x3000, // Block 0x58, offset 0x1600 - 0x1610: 0x00e6, 0x1611: 0x00e6, - 0x1612: 0x0001, 0x1613: 0x0001, 0x1614: 0x00e6, 0x1615: 0x00e6, 0x1616: 0x00e6, 0x1617: 0x00e6, - 0x1618: 0x0001, 0x1619: 0x0001, 0x161a: 0x0001, 0x161b: 0x00e6, 0x161c: 0x00e6, - 0x1621: 0x00e6, - 0x1625: 0x0001, 0x1626: 0x0001, 0x1627: 0x00e6, 0x1628: 0x00dc, 0x1629: 0x00e6, - 0x162a: 0x0001, 0x162b: 0x0001, 0x162c: 0x00dc, 0x162d: 0x00dc, 0x162e: 0x00dc, 0x162f: 0x00dc, - 0x1630: 0x00e6, + 0x1600: 0x3000, 0x1601: 0x3000, 0x1602: 0x3000, 0x1603: 0x3000, 0x1604: 0x3000, 0x1605: 0x3000, + 0x1606: 0x3000, 0x1607: 0x3000, 0x1608: 0x3000, 0x1609: 0x3000, 0x160a: 0x3000, 0x160b: 0x3000, + 0x160c: 0x3000, 0x160d: 0x3000, 0x160e: 0x3000, 0x1610: 0x3000, 0x1611: 0x3000, + 0x1612: 0x3000, 0x1613: 0x3000, 0x1614: 0x3000, 0x1615: 0x3000, 0x1616: 0x3000, 0x1617: 0x3000, + 0x1618: 0x3000, 0x1619: 0x3000, 0x161a: 0x3000, 0x161b: 0x3000, 0x161c: 0x3000, + 0x1628: 0x3000, // Block 0x59, offset 0x1640 - 0x1640: 0x3000, 0x1641: 0x3000, 0x1642: 0x3000, 0x1643: 0x3000, 0x1645: 0x3000, - 0x1646: 0x3000, 0x1647: 0x3000, 0x1649: 0x3000, 0x164a: 0x3000, 0x164b: 0x3000, - 0x164c: 0x3000, 0x164d: 0x3000, 0x164e: 0x3000, 0x164f: 0x3000, 0x1650: 0x3000, 0x1651: 0x3000, - 0x1652: 0x3000, 0x1653: 0x3000, 0x1655: 0x3000, 0x1656: 0x3000, - 0x1659: 0x3000, 0x165a: 0x3000, 0x165b: 0x3000, 0x165c: 0x3000, 0x165d: 0x3000, - 0x1660: 0x3000, 0x1661: 0x3000, 0x1662: 0x3000, - 0x1664: 0x3000, 0x1666: 0x3300, 0x1668: 0x3000, - 0x166a: 0x3300, 0x166b: 0x3300, 0x166c: 0x3000, 0x166d: 0x3000, 0x166f: 0x3000, - 0x1670: 0x3000, 0x1671: 0x3000, 0x1673: 0x3000, 0x1674: 0x3000, 0x1675: 0x3000, - 0x1676: 0x3000, 0x1677: 0x3000, 0x1678: 0x3000, 0x1679: 0x3000, 0x167b: 0x3000, - 0x167c: 0x3000, 0x167d: 0x3000, 0x167e: 0x3000, 0x167f: 0x3000, + 0x1650: 0x00e6, 0x1651: 0x00e6, + 0x1652: 0x0001, 0x1653: 0x0001, 0x1654: 0x00e6, 0x1655: 0x00e6, 0x1656: 0x00e6, 0x1657: 0x00e6, + 0x1658: 0x0001, 0x1659: 0x0001, 0x165a: 0x0001, 0x165b: 0x00e6, 0x165c: 0x00e6, + 0x1661: 0x00e6, + 0x1665: 0x0001, 0x1666: 0x0001, 0x1667: 0x00e6, 0x1668: 0x00dc, 0x1669: 0x00e6, + 0x166a: 0x0001, 0x166b: 0x0001, 0x166c: 0x00dc, 0x166d: 0x00dc, 0x166e: 0x00dc, 0x166f: 0x00dc, + 0x1670: 0x00e6, // Block 0x5a, offset 0x1680 - 0x1680: 0x3000, 0x1685: 0x3000, - 0x1686: 0x3000, 0x1687: 0x3000, 0x1688: 0x3000, 0x1689: 0x3000, - 0x1690: 0x3000, 0x1691: 0x3000, - 0x1692: 0x3000, 0x1693: 0x3000, 0x1694: 0x3000, 0x1695: 0x3000, 0x1696: 0x3000, 0x1697: 0x3000, - 0x1698: 0x3000, 0x1699: 0x3000, 0x169a: 0x3000, 0x169b: 0x3000, 0x169c: 0x3000, 0x169d: 0x3000, - 0x169e: 0x3000, 0x169f: 0x3000, 0x16a0: 0x3000, 0x16a1: 0x3000, 0x16a2: 0x3000, 0x16a3: 0x3000, - 0x16a4: 0x3000, 0x16a5: 0x3000, 0x16a6: 0x3000, 0x16a7: 0x3000, 0x16a8: 0x3000, 0x16a9: 0x3000, - 0x16aa: 0x3000, 0x16ab: 0x3000, 0x16ac: 0x3000, 0x16ad: 0x3000, 0x16ae: 0x3000, 0x16af: 0x3000, - 0x16b0: 0x3000, 0x16b1: 0x3000, 0x16b2: 0x3000, 0x16b3: 0x3000, 0x16b4: 0x3000, 0x16b5: 0x3000, - 0x16b6: 0x3000, 0x16b7: 0x3000, 0x16b8: 0x3000, 0x16b9: 0x3000, 0x16ba: 0x3000, 0x16bb: 0x3000, + 0x1680: 0x3000, 0x1681: 0x3000, 0x1682: 0x3000, 0x1683: 0x3000, 0x1685: 0x3000, + 0x1686: 0x3000, 0x1687: 0x3000, 0x1689: 0x3000, 0x168a: 0x3000, 0x168b: 0x3000, + 0x168c: 0x3000, 0x168d: 0x3000, 0x168e: 0x3000, 0x168f: 0x3000, 0x1690: 0x3000, 0x1691: 0x3000, + 0x1692: 0x3000, 0x1693: 0x3000, 0x1695: 0x3000, 0x1696: 0x3000, + 0x1699: 0x3000, 0x169a: 0x3000, 0x169b: 0x3000, 0x169c: 0x3000, 0x169d: 0x3000, + 0x16a0: 0x3000, 0x16a1: 0x3000, 0x16a2: 0x3000, + 0x16a4: 0x3000, 0x16a6: 0x3300, 0x16a8: 0x3000, + 0x16aa: 0x3300, 0x16ab: 0x3300, 0x16ac: 0x3000, 0x16ad: 0x3000, 0x16af: 0x3000, + 0x16b0: 0x3000, 0x16b1: 0x3000, 0x16b3: 0x3000, 0x16b4: 0x3000, 0x16b5: 0x3000, + 0x16b6: 0x3000, 0x16b7: 0x3000, 0x16b8: 0x3000, 0x16b9: 0x3000, 0x16bb: 0x3000, 0x16bc: 0x3000, 0x16bd: 0x3000, 0x16be: 0x3000, 0x16bf: 0x3000, // Block 0x5b, offset 0x16c0 - 0x16c9: 0x3000, - 0x16d0: 0x8800, - 0x16d2: 0x8800, 0x16d4: 0x8800, - 0x16da: 0x1100, 0x16db: 0x1100, - 0x16ee: 0x1100, + 0x16c0: 0x3000, 0x16c5: 0x3000, + 0x16c6: 0x3000, 0x16c7: 0x3000, 0x16c8: 0x3000, 0x16c9: 0x3000, + 0x16d0: 0x3000, 0x16d1: 0x3000, + 0x16d2: 0x3000, 0x16d3: 0x3000, 0x16d4: 0x3000, 0x16d5: 0x3000, 0x16d6: 0x3000, 0x16d7: 0x3000, + 0x16d8: 0x3000, 0x16d9: 0x3000, 0x16da: 0x3000, 0x16db: 0x3000, 0x16dc: 0x3000, 0x16dd: 0x3000, + 0x16de: 0x3000, 0x16df: 0x3000, 0x16e0: 0x3000, 0x16e1: 0x3000, 0x16e2: 0x3000, 0x16e3: 0x3000, + 0x16e4: 0x3000, 0x16e5: 0x3000, 0x16e6: 0x3000, 0x16e7: 0x3000, 0x16e8: 0x3000, 0x16e9: 0x3000, + 0x16ea: 0x3000, 0x16eb: 0x3000, 0x16ec: 0x3000, 0x16ed: 0x3000, 0x16ee: 0x3000, 0x16ef: 0x3000, + 0x16f0: 0x3000, 0x16f1: 0x3000, 0x16f2: 0x3000, 0x16f3: 0x3000, 0x16f4: 0x3000, 0x16f5: 0x3000, + 0x16f6: 0x3000, 0x16f7: 0x3000, 0x16f8: 0x3000, 0x16f9: 0x3000, 0x16fa: 0x3000, 0x16fb: 0x3000, + 0x16fc: 0x3000, 0x16fd: 0x3000, 0x16fe: 0x3000, 0x16ff: 0x3000, // Block 0x5c, offset 0x1700 - 0x170d: 0x1100, 0x170e: 0x1100, 0x170f: 0x1100, 0x1710: 0x8800, + 0x1709: 0x3000, + 0x1710: 0x8800, 0x1712: 0x8800, 0x1714: 0x8800, + 0x171a: 0x1100, 0x171b: 0x1100, + 0x172e: 0x1100, // Block 0x5d, offset 0x1740 - 0x1743: 0x8800, 0x1744: 0x1100, - 0x1748: 0x8800, 0x1749: 0x1100, 0x174b: 0x8800, - 0x174c: 0x1100, - 0x1763: 0x8800, - 0x1764: 0x1100, 0x1765: 0x8800, 0x1766: 0x1100, - 0x176c: 0x3000, 0x176d: 0x3000, 0x176f: 0x3000, - 0x1770: 0x3000, - 0x177c: 0x8800, + 0x174d: 0x1100, 0x174e: 0x1100, 0x174f: 0x1100, 0x1750: 0x8800, + 0x1752: 0x8800, 0x1754: 0x8800, // Block 0x5e, offset 0x1780 - 0x1781: 0x1100, 0x1783: 0x8800, 0x1784: 0x1100, 0x1785: 0x8800, - 0x1787: 0x1100, 0x1788: 0x8800, 0x1789: 0x1100, - 0x178d: 0x8800, - 0x17a0: 0x1100, 0x17a1: 0x8800, 0x17a2: 0x1100, - 0x17a4: 0x8800, 0x17a5: 0x8800, - 0x17ad: 0x1100, 0x17ae: 0x1100, 0x17af: 0x1100, - 0x17b0: 0x1100, 0x17b1: 0x1100, 0x17b2: 0x8800, 0x17b3: 0x8800, 0x17b4: 0x1100, 0x17b5: 0x1100, - 0x17b6: 0x8800, 0x17b7: 0x8800, 0x17b8: 0x1100, 0x17b9: 0x1100, 0x17ba: 0x8800, 0x17bb: 0x8800, - 0x17bc: 0x8800, 0x17bd: 0x8800, + 0x1783: 0x8800, 0x1784: 0x1100, + 0x1788: 0x8800, 0x1789: 0x1100, 0x178b: 0x8800, + 0x178c: 0x1100, + 0x17a3: 0x8800, + 0x17a4: 0x1100, 0x17a5: 0x8800, 0x17a6: 0x1100, + 0x17ac: 0x3000, 0x17ad: 0x3000, 0x17af: 0x3000, + 0x17b0: 0x3000, + 0x17bc: 0x8800, // Block 0x5f, offset 0x17c0 - 0x17c0: 0x1100, 0x17c1: 0x1100, 0x17c2: 0x8800, 0x17c3: 0x8800, 0x17c4: 0x1100, 0x17c5: 0x1100, - 0x17c6: 0x8800, 0x17c7: 0x8800, 0x17c8: 0x1100, 0x17c9: 0x1100, - 0x17d1: 0x8800, - 0x17d2: 0x8800, - 0x17e2: 0x8800, - 0x17e8: 0x8800, 0x17e9: 0x8800, - 0x17eb: 0x8800, 0x17ec: 0x1100, 0x17ed: 0x1100, 0x17ee: 0x1100, 0x17ef: 0x1100, - 0x17f2: 0x8800, 0x17f3: 0x8800, 0x17f4: 0x8800, 0x17f5: 0x8800, + 0x17c1: 0x1100, 0x17c3: 0x8800, 0x17c4: 0x1100, 0x17c5: 0x8800, + 0x17c7: 0x1100, 0x17c8: 0x8800, 0x17c9: 0x1100, + 0x17cd: 0x8800, + 0x17e0: 0x1100, 0x17e1: 0x8800, 0x17e2: 0x1100, + 0x17e4: 0x8800, 0x17e5: 0x8800, + 0x17ed: 0x1100, 0x17ee: 0x1100, 0x17ef: 0x1100, + 0x17f0: 0x1100, 0x17f1: 0x1100, 0x17f2: 0x8800, 0x17f3: 0x8800, 0x17f4: 0x1100, 0x17f5: 0x1100, + 0x17f6: 0x8800, 0x17f7: 0x8800, 0x17f8: 0x1100, 0x17f9: 0x1100, 0x17fa: 0x8800, 0x17fb: 0x8800, + 0x17fc: 0x8800, 0x17fd: 0x8800, // Block 0x60, offset 0x1800 - 0x1820: 0x1100, 0x1821: 0x1100, 0x1822: 0x1100, 0x1823: 0x1100, - 0x182a: 0x1100, 0x182b: 0x1100, 0x182c: 0x1100, 0x182d: 0x1100, + 0x1800: 0x1100, 0x1801: 0x1100, 0x1802: 0x8800, 0x1803: 0x8800, 0x1804: 0x1100, 0x1805: 0x1100, + 0x1806: 0x8800, 0x1807: 0x8800, 0x1808: 0x1100, 0x1809: 0x1100, + 0x1811: 0x8800, + 0x1812: 0x8800, + 0x1822: 0x8800, + 0x1828: 0x8800, 0x1829: 0x8800, + 0x182b: 0x8800, 0x182c: 0x1100, 0x182d: 0x1100, 0x182e: 0x1100, 0x182f: 0x1100, + 0x1832: 0x8800, 0x1833: 0x8800, 0x1834: 0x8800, 0x1835: 0x8800, // Block 0x61, offset 0x1840 - 0x1869: 0x3300, - 0x186a: 0x3300, + 0x1860: 0x1100, 0x1861: 0x1100, 0x1862: 0x1100, 0x1863: 0x1100, + 0x186a: 0x1100, 0x186b: 0x1100, 0x186c: 0x1100, 0x186d: 0x1100, // Block 0x62, offset 0x1880 - 0x18a0: 0x3000, 0x18a1: 0x3000, 0x18a2: 0x3000, 0x18a3: 0x3000, - 0x18a4: 0x3000, 0x18a5: 0x3000, 0x18a6: 0x3000, 0x18a7: 0x3000, 0x18a8: 0x3000, 0x18a9: 0x3000, - 0x18aa: 0x3000, 0x18ab: 0x3000, 0x18ac: 0x3000, 0x18ad: 0x3000, 0x18ae: 0x3000, 0x18af: 0x3000, - 0x18b0: 0x3000, 0x18b1: 0x3000, 0x18b2: 0x3000, 0x18b3: 0x3000, 0x18b4: 0x3000, 0x18b5: 0x3000, - 0x18b6: 0x3000, 0x18b7: 0x3000, 0x18b8: 0x3000, 0x18b9: 0x3000, 0x18ba: 0x3000, 0x18bb: 0x3000, - 0x18bc: 0x3000, 0x18bd: 0x3000, 0x18be: 0x3000, 0x18bf: 0x3000, + 0x18a9: 0x3300, + 0x18aa: 0x3300, // Block 0x63, offset 0x18c0 - 0x18c0: 0x3000, 0x18c1: 0x3000, 0x18c2: 0x3000, 0x18c3: 0x3000, 0x18c4: 0x3000, 0x18c5: 0x3000, - 0x18c6: 0x3000, 0x18c7: 0x3000, 0x18c8: 0x3000, 0x18c9: 0x3000, 0x18ca: 0x3000, 0x18cb: 0x3000, - 0x18cc: 0x3000, 0x18cd: 0x3000, 0x18ce: 0x3000, 0x18cf: 0x3000, 0x18d0: 0x3000, 0x18d1: 0x3000, - 0x18d2: 0x3000, 0x18d3: 0x3000, 0x18d4: 0x3000, 0x18d5: 0x3000, 0x18d6: 0x3000, 0x18d7: 0x3000, - 0x18d8: 0x3000, 0x18d9: 0x3000, 0x18da: 0x3000, 0x18db: 0x3000, 0x18dc: 0x3000, 0x18dd: 0x3000, - 0x18de: 0x3000, 0x18df: 0x3000, 0x18e0: 0x3000, 0x18e1: 0x3000, 0x18e2: 0x3000, 0x18e3: 0x3000, + 0x18e0: 0x3000, 0x18e1: 0x3000, 0x18e2: 0x3000, 0x18e3: 0x3000, 0x18e4: 0x3000, 0x18e5: 0x3000, 0x18e6: 0x3000, 0x18e7: 0x3000, 0x18e8: 0x3000, 0x18e9: 0x3000, 0x18ea: 0x3000, 0x18eb: 0x3000, 0x18ec: 0x3000, 0x18ed: 0x3000, 0x18ee: 0x3000, 0x18ef: 0x3000, 0x18f0: 0x3000, 0x18f1: 0x3000, 0x18f2: 0x3000, 0x18f3: 0x3000, 0x18f4: 0x3000, 0x18f5: 0x3000, @@ -5998,111 +5999,111 @@ var charInfoValues = [10944]uint16{ 0x1918: 0x3000, 0x1919: 0x3000, 0x191a: 0x3000, 0x191b: 0x3000, 0x191c: 0x3000, 0x191d: 0x3000, 0x191e: 0x3000, 0x191f: 0x3000, 0x1920: 0x3000, 0x1921: 0x3000, 0x1922: 0x3000, 0x1923: 0x3000, 0x1924: 0x3000, 0x1925: 0x3000, 0x1926: 0x3000, 0x1927: 0x3000, 0x1928: 0x3000, 0x1929: 0x3000, - 0x192a: 0x3000, + 0x192a: 0x3000, 0x192b: 0x3000, 0x192c: 0x3000, 0x192d: 0x3000, 0x192e: 0x3000, 0x192f: 0x3000, + 0x1930: 0x3000, 0x1931: 0x3000, 0x1932: 0x3000, 0x1933: 0x3000, 0x1934: 0x3000, 0x1935: 0x3000, + 0x1936: 0x3000, 0x1937: 0x3000, 0x1938: 0x3000, 0x1939: 0x3000, 0x193a: 0x3000, 0x193b: 0x3000, + 0x193c: 0x3000, 0x193d: 0x3000, 0x193e: 0x3000, 0x193f: 0x3000, // Block 0x65, offset 0x1940 - 0x194c: 0x3000, + 0x1940: 0x3000, 0x1941: 0x3000, 0x1942: 0x3000, 0x1943: 0x3000, 0x1944: 0x3000, 0x1945: 0x3000, + 0x1946: 0x3000, 0x1947: 0x3000, 0x1948: 0x3000, 0x1949: 0x3000, 0x194a: 0x3000, 0x194b: 0x3000, + 0x194c: 0x3000, 0x194d: 0x3000, 0x194e: 0x3000, 0x194f: 0x3000, 0x1950: 0x3000, 0x1951: 0x3000, + 0x1952: 0x3000, 0x1953: 0x3000, 0x1954: 0x3000, 0x1955: 0x3000, 0x1956: 0x3000, 0x1957: 0x3000, + 0x1958: 0x3000, 0x1959: 0x3000, 0x195a: 0x3000, 0x195b: 0x3000, 0x195c: 0x3000, 0x195d: 0x3000, + 0x195e: 0x3000, 0x195f: 0x3000, 0x1960: 0x3000, 0x1961: 0x3000, 0x1962: 0x3000, 0x1963: 0x3000, + 0x1964: 0x3000, 0x1965: 0x3000, 0x1966: 0x3000, 0x1967: 0x3000, 0x1968: 0x3000, 0x1969: 0x3000, + 0x196a: 0x3000, // Block 0x66, offset 0x1980 - 0x19b4: 0x3000, 0x19b5: 0x3000, - 0x19b6: 0x3000, + 0x198c: 0x3000, // Block 0x67, offset 0x19c0 - 0x19dc: 0x3300, + 0x19f4: 0x3000, 0x19f5: 0x3000, + 0x19f6: 0x3000, // Block 0x68, offset 0x1a00 - 0x1a3c: 0x3000, 0x1a3d: 0x3000, + 0x1a1c: 0x3300, // Block 0x69, offset 0x1a40 - 0x1a6f: 0x00e6, - 0x1a70: 0x00e6, 0x1a71: 0x00e6, + 0x1a7c: 0x3000, 0x1a7d: 0x3000, // Block 0x6a, offset 0x1a80 - 0x1aaf: 0x3000, - 0x1abf: 0x0009, + 0x1aaf: 0x00e6, + 0x1ab0: 0x00e6, 0x1ab1: 0x00e6, // Block 0x6b, offset 0x1ac0 - 0x1ae0: 0x00e6, 0x1ae1: 0x00e6, 0x1ae2: 0x00e6, 0x1ae3: 0x00e6, - 0x1ae4: 0x00e6, 0x1ae5: 0x00e6, 0x1ae6: 0x00e6, 0x1ae7: 0x00e6, 0x1ae8: 0x00e6, 0x1ae9: 0x00e6, - 0x1aea: 0x00e6, 0x1aeb: 0x00e6, 0x1aec: 0x00e6, 0x1aed: 0x00e6, 0x1aee: 0x00e6, 0x1aef: 0x00e6, - 0x1af0: 0x00e6, 0x1af1: 0x00e6, 0x1af2: 0x00e6, 0x1af3: 0x00e6, 0x1af4: 0x00e6, 0x1af5: 0x00e6, - 0x1af6: 0x00e6, 0x1af7: 0x00e6, 0x1af8: 0x00e6, 0x1af9: 0x00e6, 0x1afa: 0x00e6, 0x1afb: 0x00e6, - 0x1afc: 0x00e6, 0x1afd: 0x00e6, 0x1afe: 0x00e6, 0x1aff: 0x00e6, + 0x1aef: 0x3000, + 0x1aff: 0x0009, // Block 0x6c, offset 0x1b00 - 0x1b1f: 0x3000, + 0x1b20: 0x00e6, 0x1b21: 0x00e6, 0x1b22: 0x00e6, 0x1b23: 0x00e6, + 0x1b24: 0x00e6, 0x1b25: 0x00e6, 0x1b26: 0x00e6, 0x1b27: 0x00e6, 0x1b28: 0x00e6, 0x1b29: 0x00e6, + 0x1b2a: 0x00e6, 0x1b2b: 0x00e6, 0x1b2c: 0x00e6, 0x1b2d: 0x00e6, 0x1b2e: 0x00e6, 0x1b2f: 0x00e6, + 0x1b30: 0x00e6, 0x1b31: 0x00e6, 0x1b32: 0x00e6, 0x1b33: 0x00e6, 0x1b34: 0x00e6, 0x1b35: 0x00e6, + 0x1b36: 0x00e6, 0x1b37: 0x00e6, 0x1b38: 0x00e6, 0x1b39: 0x00e6, 0x1b3a: 0x00e6, 0x1b3b: 0x00e6, + 0x1b3c: 0x00e6, 0x1b3d: 0x00e6, 0x1b3e: 0x00e6, 0x1b3f: 0x00e6, // Block 0x6d, offset 0x1b40 - 0x1b73: 0x3000, + 0x1b5f: 0x3000, // Block 0x6e, offset 0x1b80 - 0x1b80: 0x3000, 0x1b81: 0x3000, 0x1b82: 0x3000, 0x1b83: 0x3000, 0x1b84: 0x3000, 0x1b85: 0x3000, - 0x1b86: 0x3000, 0x1b87: 0x3000, 0x1b88: 0x3000, 0x1b89: 0x3000, 0x1b8a: 0x3000, 0x1b8b: 0x3000, - 0x1b8c: 0x3000, 0x1b8d: 0x3000, 0x1b8e: 0x3000, 0x1b8f: 0x3000, 0x1b90: 0x3000, 0x1b91: 0x3000, - 0x1b92: 0x3000, 0x1b93: 0x3000, 0x1b94: 0x3000, 0x1b95: 0x3000, + 0x1bb3: 0x3000, // Block 0x6f, offset 0x1bc0 - 0x1bc0: 0x3000, - 0x1bea: 0x00da, 0x1beb: 0x00e4, 0x1bec: 0x00e8, 0x1bed: 0x00de, 0x1bee: 0x00e0, 0x1bef: 0x00e0, - 0x1bf6: 0x3000, 0x1bf8: 0x3000, 0x1bf9: 0x3000, 0x1bfa: 0x3000, + 0x1bc0: 0x3000, 0x1bc1: 0x3000, 0x1bc2: 0x3000, 0x1bc3: 0x3000, 0x1bc4: 0x3000, 0x1bc5: 0x3000, + 0x1bc6: 0x3000, 0x1bc7: 0x3000, 0x1bc8: 0x3000, 0x1bc9: 0x3000, 0x1bca: 0x3000, 0x1bcb: 0x3000, + 0x1bcc: 0x3000, 0x1bcd: 0x3000, 0x1bce: 0x3000, 0x1bcf: 0x3000, 0x1bd0: 0x3000, 0x1bd1: 0x3000, + 0x1bd2: 0x3000, 0x1bd3: 0x3000, 0x1bd4: 0x3000, 0x1bd5: 0x3000, // Block 0x70, offset 0x1c00 - 0x1c06: 0x8800, 0x1c0b: 0x8800, - 0x1c0c: 0x1100, 0x1c0d: 0x8800, 0x1c0e: 0x1100, 0x1c0f: 0x8800, 0x1c10: 0x1100, 0x1c11: 0x8800, - 0x1c12: 0x1100, 0x1c13: 0x8800, 0x1c14: 0x1100, 0x1c15: 0x8800, 0x1c16: 0x1100, 0x1c17: 0x8800, - 0x1c18: 0x1100, 0x1c19: 0x8800, 0x1c1a: 0x1100, 0x1c1b: 0x8800, 0x1c1c: 0x1100, 0x1c1d: 0x8800, - 0x1c1e: 0x1100, 0x1c1f: 0x8800, 0x1c20: 0x1100, 0x1c21: 0x8800, 0x1c22: 0x1100, - 0x1c24: 0x8800, 0x1c25: 0x1100, 0x1c26: 0x8800, 0x1c27: 0x1100, 0x1c28: 0x8800, 0x1c29: 0x1100, - 0x1c2f: 0x8800, - 0x1c30: 0x1100, 0x1c31: 0x1100, 0x1c32: 0x8800, 0x1c33: 0x1100, 0x1c34: 0x1100, 0x1c35: 0x8800, - 0x1c36: 0x1100, 0x1c37: 0x1100, 0x1c38: 0x8800, 0x1c39: 0x1100, 0x1c3a: 0x1100, 0x1c3b: 0x8800, - 0x1c3c: 0x1100, 0x1c3d: 0x1100, + 0x1c00: 0x3000, + 0x1c2a: 0x00da, 0x1c2b: 0x00e4, 0x1c2c: 0x00e8, 0x1c2d: 0x00de, 0x1c2e: 0x00e0, 0x1c2f: 0x00e0, + 0x1c36: 0x3000, 0x1c38: 0x3000, 0x1c39: 0x3000, 0x1c3a: 0x3000, // Block 0x71, offset 0x1c40 - 0x1c54: 0x1100, - 0x1c59: 0x6608, 0x1c5a: 0x6608, 0x1c5b: 0x3000, 0x1c5c: 0x3000, 0x1c5d: 0x8800, - 0x1c5e: 0x1100, 0x1c5f: 0x3000, - 0x1c66: 0x8800, - 0x1c6b: 0x8800, 0x1c6c: 0x1100, 0x1c6d: 0x8800, 0x1c6e: 0x1100, 0x1c6f: 0x8800, - 0x1c70: 0x1100, 0x1c71: 0x8800, 0x1c72: 0x1100, 0x1c73: 0x8800, 0x1c74: 0x1100, 0x1c75: 0x8800, - 0x1c76: 0x1100, 0x1c77: 0x8800, 0x1c78: 0x1100, 0x1c79: 0x8800, 0x1c7a: 0x1100, 0x1c7b: 0x8800, - 0x1c7c: 0x1100, 0x1c7d: 0x8800, 0x1c7e: 0x1100, 0x1c7f: 0x8800, + 0x1c46: 0x8800, 0x1c4b: 0x8800, + 0x1c4c: 0x1100, 0x1c4d: 0x8800, 0x1c4e: 0x1100, 0x1c4f: 0x8800, 0x1c50: 0x1100, 0x1c51: 0x8800, + 0x1c52: 0x1100, 0x1c53: 0x8800, 0x1c54: 0x1100, 0x1c55: 0x8800, 0x1c56: 0x1100, 0x1c57: 0x8800, + 0x1c58: 0x1100, 0x1c59: 0x8800, 0x1c5a: 0x1100, 0x1c5b: 0x8800, 0x1c5c: 0x1100, 0x1c5d: 0x8800, + 0x1c5e: 0x1100, 0x1c5f: 0x8800, 0x1c60: 0x1100, 0x1c61: 0x8800, 0x1c62: 0x1100, + 0x1c64: 0x8800, 0x1c65: 0x1100, 0x1c66: 0x8800, 0x1c67: 0x1100, 0x1c68: 0x8800, 0x1c69: 0x1100, + 0x1c6f: 0x8800, + 0x1c70: 0x1100, 0x1c71: 0x1100, 0x1c72: 0x8800, 0x1c73: 0x1100, 0x1c74: 0x1100, 0x1c75: 0x8800, + 0x1c76: 0x1100, 0x1c77: 0x1100, 0x1c78: 0x8800, 0x1c79: 0x1100, 0x1c7a: 0x1100, 0x1c7b: 0x8800, + 0x1c7c: 0x1100, 0x1c7d: 0x1100, // Block 0x72, offset 0x1c80 - 0x1c80: 0x1100, 0x1c81: 0x8800, 0x1c82: 0x1100, 0x1c84: 0x8800, 0x1c85: 0x1100, - 0x1c86: 0x8800, 0x1c87: 0x1100, 0x1c88: 0x8800, 0x1c89: 0x1100, - 0x1c8f: 0x8800, 0x1c90: 0x1100, 0x1c91: 0x1100, - 0x1c92: 0x8800, 0x1c93: 0x1100, 0x1c94: 0x1100, 0x1c95: 0x8800, 0x1c96: 0x1100, 0x1c97: 0x1100, - 0x1c98: 0x8800, 0x1c99: 0x1100, 0x1c9a: 0x1100, 0x1c9b: 0x8800, 0x1c9c: 0x1100, 0x1c9d: 0x1100, - 0x1caf: 0x8800, - 0x1cb0: 0x8800, 0x1cb1: 0x8800, 0x1cb2: 0x8800, 0x1cb4: 0x1100, - 0x1cb7: 0x1100, 0x1cb8: 0x1100, 0x1cb9: 0x1100, 0x1cba: 0x1100, - 0x1cbd: 0x8800, 0x1cbe: 0x1100, 0x1cbf: 0x3000, + 0x1c94: 0x1100, + 0x1c99: 0x6608, 0x1c9a: 0x6608, 0x1c9b: 0x3000, 0x1c9c: 0x3000, 0x1c9d: 0x8800, + 0x1c9e: 0x1100, 0x1c9f: 0x3000, + 0x1ca6: 0x8800, + 0x1cab: 0x8800, 0x1cac: 0x1100, 0x1cad: 0x8800, 0x1cae: 0x1100, 0x1caf: 0x8800, + 0x1cb0: 0x1100, 0x1cb1: 0x8800, 0x1cb2: 0x1100, 0x1cb3: 0x8800, 0x1cb4: 0x1100, 0x1cb5: 0x8800, + 0x1cb6: 0x1100, 0x1cb7: 0x8800, 0x1cb8: 0x1100, 0x1cb9: 0x8800, 0x1cba: 0x1100, 0x1cbb: 0x8800, + 0x1cbc: 0x1100, 0x1cbd: 0x8800, 0x1cbe: 0x1100, 0x1cbf: 0x8800, // Block 0x73, offset 0x1cc0 - 0x1cf1: 0x3000, 0x1cf2: 0x3000, 0x1cf3: 0x3000, 0x1cf4: 0x3000, 0x1cf5: 0x3000, - 0x1cf6: 0x3000, 0x1cf7: 0x3000, 0x1cf8: 0x3000, 0x1cf9: 0x3000, 0x1cfa: 0x3000, 0x1cfb: 0x3000, - 0x1cfc: 0x3000, 0x1cfd: 0x3000, 0x1cfe: 0x3000, 0x1cff: 0x3000, + 0x1cc0: 0x1100, 0x1cc1: 0x8800, 0x1cc2: 0x1100, 0x1cc4: 0x8800, 0x1cc5: 0x1100, + 0x1cc6: 0x8800, 0x1cc7: 0x1100, 0x1cc8: 0x8800, 0x1cc9: 0x1100, + 0x1ccf: 0x8800, 0x1cd0: 0x1100, 0x1cd1: 0x1100, + 0x1cd2: 0x8800, 0x1cd3: 0x1100, 0x1cd4: 0x1100, 0x1cd5: 0x8800, 0x1cd6: 0x1100, 0x1cd7: 0x1100, + 0x1cd8: 0x8800, 0x1cd9: 0x1100, 0x1cda: 0x1100, 0x1cdb: 0x8800, 0x1cdc: 0x1100, 0x1cdd: 0x1100, + 0x1cef: 0x8800, + 0x1cf0: 0x8800, 0x1cf1: 0x8800, 0x1cf2: 0x8800, 0x1cf4: 0x1100, + 0x1cf7: 0x1100, 0x1cf8: 0x1100, 0x1cf9: 0x1100, 0x1cfa: 0x1100, + 0x1cfd: 0x8800, 0x1cfe: 0x1100, 0x1cff: 0x3000, // Block 0x74, offset 0x1d00 - 0x1d00: 0x3000, 0x1d01: 0x3000, 0x1d02: 0x3000, 0x1d03: 0x3000, 0x1d04: 0x3000, 0x1d05: 0x3000, - 0x1d06: 0x3000, 0x1d07: 0x3000, 0x1d08: 0x3000, 0x1d09: 0x3000, 0x1d0a: 0x3000, 0x1d0b: 0x3000, - 0x1d0c: 0x3000, 0x1d0d: 0x3000, 0x1d0e: 0x3000, - 0x1d12: 0x3000, 0x1d13: 0x3000, 0x1d14: 0x3000, 0x1d15: 0x3000, 0x1d16: 0x3000, 0x1d17: 0x3000, - 0x1d18: 0x3000, 0x1d19: 0x3000, 0x1d1a: 0x3000, 0x1d1b: 0x3000, 0x1d1c: 0x3000, 0x1d1d: 0x3000, - 0x1d1e: 0x3000, 0x1d1f: 0x3000, + 0x1d31: 0x3000, 0x1d32: 0x3000, 0x1d33: 0x3000, 0x1d34: 0x3000, 0x1d35: 0x3000, + 0x1d36: 0x3000, 0x1d37: 0x3000, 0x1d38: 0x3000, 0x1d39: 0x3000, 0x1d3a: 0x3000, 0x1d3b: 0x3000, + 0x1d3c: 0x3000, 0x1d3d: 0x3000, 0x1d3e: 0x3000, 0x1d3f: 0x3000, // Block 0x75, offset 0x1d40 0x1d40: 0x3000, 0x1d41: 0x3000, 0x1d42: 0x3000, 0x1d43: 0x3000, 0x1d44: 0x3000, 0x1d45: 0x3000, 0x1d46: 0x3000, 0x1d47: 0x3000, 0x1d48: 0x3000, 0x1d49: 0x3000, 0x1d4a: 0x3000, 0x1d4b: 0x3000, - 0x1d4c: 0x3000, 0x1d4d: 0x3000, 0x1d4e: 0x3000, 0x1d4f: 0x3000, 0x1d50: 0x3000, 0x1d51: 0x3000, + 0x1d4c: 0x3000, 0x1d4d: 0x3000, 0x1d4e: 0x3000, 0x1d52: 0x3000, 0x1d53: 0x3000, 0x1d54: 0x3000, 0x1d55: 0x3000, 0x1d56: 0x3000, 0x1d57: 0x3000, 0x1d58: 0x3000, 0x1d59: 0x3000, 0x1d5a: 0x3000, 0x1d5b: 0x3000, 0x1d5c: 0x3000, 0x1d5d: 0x3000, - 0x1d5e: 0x3000, 0x1d60: 0x3000, 0x1d61: 0x3000, 0x1d62: 0x3000, 0x1d63: 0x3000, - 0x1d64: 0x3000, 0x1d65: 0x3000, 0x1d66: 0x3000, 0x1d67: 0x3000, 0x1d68: 0x3000, 0x1d69: 0x3000, - 0x1d6a: 0x3000, 0x1d6b: 0x3000, 0x1d6c: 0x3000, 0x1d6d: 0x3000, 0x1d6e: 0x3000, 0x1d6f: 0x3000, - 0x1d70: 0x3000, 0x1d71: 0x3000, 0x1d72: 0x3000, 0x1d73: 0x3000, 0x1d74: 0x3000, 0x1d75: 0x3000, - 0x1d76: 0x3000, 0x1d77: 0x3000, 0x1d78: 0x3000, 0x1d79: 0x3000, 0x1d7a: 0x3000, 0x1d7b: 0x3000, - 0x1d7c: 0x3000, 0x1d7d: 0x3000, 0x1d7e: 0x3000, 0x1d7f: 0x3000, + 0x1d5e: 0x3000, 0x1d5f: 0x3000, // Block 0x76, offset 0x1d80 0x1d80: 0x3000, 0x1d81: 0x3000, 0x1d82: 0x3000, 0x1d83: 0x3000, 0x1d84: 0x3000, 0x1d85: 0x3000, - 0x1d86: 0x3000, 0x1d87: 0x3000, - 0x1d90: 0x3000, 0x1d91: 0x3000, + 0x1d86: 0x3000, 0x1d87: 0x3000, 0x1d88: 0x3000, 0x1d89: 0x3000, 0x1d8a: 0x3000, 0x1d8b: 0x3000, + 0x1d8c: 0x3000, 0x1d8d: 0x3000, 0x1d8e: 0x3000, 0x1d8f: 0x3000, 0x1d90: 0x3000, 0x1d91: 0x3000, 0x1d92: 0x3000, 0x1d93: 0x3000, 0x1d94: 0x3000, 0x1d95: 0x3000, 0x1d96: 0x3000, 0x1d97: 0x3000, 0x1d98: 0x3000, 0x1d99: 0x3000, 0x1d9a: 0x3000, 0x1d9b: 0x3000, 0x1d9c: 0x3000, 0x1d9d: 0x3000, - 0x1d9e: 0x3000, 0x1d9f: 0x3000, 0x1da0: 0x3000, 0x1da1: 0x3000, 0x1da2: 0x3000, 0x1da3: 0x3000, + 0x1d9e: 0x3000, 0x1da0: 0x3000, 0x1da1: 0x3000, 0x1da2: 0x3000, 0x1da3: 0x3000, 0x1da4: 0x3000, 0x1da5: 0x3000, 0x1da6: 0x3000, 0x1da7: 0x3000, 0x1da8: 0x3000, 0x1da9: 0x3000, 0x1daa: 0x3000, 0x1dab: 0x3000, 0x1dac: 0x3000, 0x1dad: 0x3000, 0x1dae: 0x3000, 0x1daf: 0x3000, 0x1db0: 0x3000, 0x1db1: 0x3000, 0x1db2: 0x3000, 0x1db3: 0x3000, 0x1db4: 0x3000, 0x1db5: 0x3000, 0x1db6: 0x3000, 0x1db7: 0x3000, 0x1db8: 0x3000, 0x1db9: 0x3000, 0x1dba: 0x3000, 0x1dbb: 0x3000, - 0x1dbc: 0x3000, 0x1dbd: 0x3000, 0x1dbe: 0x3000, + 0x1dbc: 0x3000, 0x1dbd: 0x3000, 0x1dbe: 0x3000, 0x1dbf: 0x3000, // Block 0x77, offset 0x1dc0 0x1dc0: 0x3000, 0x1dc1: 0x3000, 0x1dc2: 0x3000, 0x1dc3: 0x3000, 0x1dc4: 0x3000, 0x1dc5: 0x3000, - 0x1dc6: 0x3000, 0x1dc7: 0x3000, 0x1dc8: 0x3000, 0x1dc9: 0x3000, 0x1dca: 0x3000, 0x1dcb: 0x3000, - 0x1dcc: 0x3000, 0x1dcd: 0x3000, 0x1dce: 0x3000, 0x1dcf: 0x3000, 0x1dd0: 0x3000, 0x1dd1: 0x3000, + 0x1dc6: 0x3000, 0x1dc7: 0x3000, + 0x1dd0: 0x3000, 0x1dd1: 0x3000, 0x1dd2: 0x3000, 0x1dd3: 0x3000, 0x1dd4: 0x3000, 0x1dd5: 0x3000, 0x1dd6: 0x3000, 0x1dd7: 0x3000, 0x1dd8: 0x3000, 0x1dd9: 0x3000, 0x1dda: 0x3000, 0x1ddb: 0x3000, 0x1ddc: 0x3000, 0x1ddd: 0x3000, 0x1dde: 0x3000, 0x1ddf: 0x3000, 0x1de0: 0x3000, 0x1de1: 0x3000, 0x1de2: 0x3000, 0x1de3: 0x3000, @@ -6112,48 +6113,48 @@ var charInfoValues = [10944]uint16{ 0x1df6: 0x3000, 0x1df7: 0x3000, 0x1df8: 0x3000, 0x1df9: 0x3000, 0x1dfa: 0x3000, 0x1dfb: 0x3000, 0x1dfc: 0x3000, 0x1dfd: 0x3000, 0x1dfe: 0x3000, // Block 0x78, offset 0x1e00 - 0x1e2f: 0x00e6, - 0x1e3c: 0x00e6, 0x1e3d: 0x00e6, + 0x1e00: 0x3000, 0x1e01: 0x3000, 0x1e02: 0x3000, 0x1e03: 0x3000, 0x1e04: 0x3000, 0x1e05: 0x3000, + 0x1e06: 0x3000, 0x1e07: 0x3000, 0x1e08: 0x3000, 0x1e09: 0x3000, 0x1e0a: 0x3000, 0x1e0b: 0x3000, + 0x1e0c: 0x3000, 0x1e0d: 0x3000, 0x1e0e: 0x3000, 0x1e0f: 0x3000, 0x1e10: 0x3000, 0x1e11: 0x3000, + 0x1e12: 0x3000, 0x1e13: 0x3000, 0x1e14: 0x3000, 0x1e15: 0x3000, 0x1e16: 0x3000, 0x1e17: 0x3000, + 0x1e18: 0x3000, 0x1e19: 0x3000, 0x1e1a: 0x3000, 0x1e1b: 0x3000, 0x1e1c: 0x3000, 0x1e1d: 0x3000, + 0x1e1e: 0x3000, 0x1e1f: 0x3000, 0x1e20: 0x3000, 0x1e21: 0x3000, 0x1e22: 0x3000, 0x1e23: 0x3000, + 0x1e24: 0x3000, 0x1e25: 0x3000, 0x1e26: 0x3000, 0x1e27: 0x3000, 0x1e28: 0x3000, 0x1e29: 0x3000, + 0x1e2a: 0x3000, 0x1e2b: 0x3000, 0x1e2c: 0x3000, 0x1e2d: 0x3000, 0x1e2e: 0x3000, 0x1e2f: 0x3000, + 0x1e30: 0x3000, 0x1e31: 0x3000, 0x1e32: 0x3000, 0x1e33: 0x3000, 0x1e34: 0x3000, 0x1e35: 0x3000, + 0x1e36: 0x3000, 0x1e37: 0x3000, 0x1e38: 0x3000, 0x1e39: 0x3000, 0x1e3a: 0x3000, 0x1e3b: 0x3000, + 0x1e3c: 0x3000, 0x1e3d: 0x3000, 0x1e3e: 0x3000, // Block 0x79, offset 0x1e40 - 0x1e70: 0x00e6, 0x1e71: 0x00e6, + 0x1e6f: 0x00e6, + 0x1e7c: 0x00e6, 0x1e7d: 0x00e6, // Block 0x7a, offset 0x1e80 - 0x1eb0: 0x3000, + 0x1eb0: 0x00e6, 0x1eb1: 0x00e6, // Block 0x7b, offset 0x1ec0 - 0x1ec6: 0x0009, + 0x1ef0: 0x3000, // Block 0x7c, offset 0x1f00 - 0x1f04: 0x0009, - 0x1f20: 0x00e6, 0x1f21: 0x00e6, 0x1f22: 0x00e6, 0x1f23: 0x00e6, - 0x1f24: 0x00e6, 0x1f25: 0x00e6, 0x1f26: 0x00e6, 0x1f27: 0x00e6, 0x1f28: 0x00e6, 0x1f29: 0x00e6, - 0x1f2a: 0x00e6, 0x1f2b: 0x00e6, 0x1f2c: 0x00e6, 0x1f2d: 0x00e6, 0x1f2e: 0x00e6, 0x1f2f: 0x00e6, - 0x1f30: 0x00e6, 0x1f31: 0x00e6, + 0x1f06: 0x0009, // Block 0x7d, offset 0x1f40 - 0x1f6b: 0x00dc, 0x1f6c: 0x00dc, 0x1f6d: 0x00dc, + 0x1f44: 0x0009, + 0x1f60: 0x00e6, 0x1f61: 0x00e6, 0x1f62: 0x00e6, 0x1f63: 0x00e6, + 0x1f64: 0x00e6, 0x1f65: 0x00e6, 0x1f66: 0x00e6, 0x1f67: 0x00e6, 0x1f68: 0x00e6, 0x1f69: 0x00e6, + 0x1f6a: 0x00e6, 0x1f6b: 0x00e6, 0x1f6c: 0x00e6, 0x1f6d: 0x00e6, 0x1f6e: 0x00e6, 0x1f6f: 0x00e6, + 0x1f70: 0x00e6, 0x1f71: 0x00e6, // Block 0x7e, offset 0x1f80 - 0x1f93: 0x0009, + 0x1fab: 0x00dc, 0x1fac: 0x00dc, 0x1fad: 0x00dc, // Block 0x7f, offset 0x1fc0 - 0x1ff3: 0x0007, + 0x1fd3: 0x0009, // Block 0x80, offset 0x2000 - 0x2000: 0x0009, + 0x2033: 0x0007, // Block 0x81, offset 0x2040 - 0x2070: 0x00e6, 0x2072: 0x00e6, 0x2073: 0x00e6, 0x2074: 0x00dc, - 0x2077: 0x00e6, 0x2078: 0x00e6, - 0x207e: 0x00e6, 0x207f: 0x00e6, + 0x2040: 0x0009, // Block 0x82, offset 0x2080 - 0x2081: 0x00e6, + 0x20b0: 0x00e6, 0x20b2: 0x00e6, 0x20b3: 0x00e6, 0x20b4: 0x00dc, + 0x20b7: 0x00e6, 0x20b8: 0x00e6, + 0x20be: 0x00e6, 0x20bf: 0x00e6, // Block 0x83, offset 0x20c0 - 0x20ed: 0x0009, + 0x20c1: 0x00e6, // Block 0x84, offset 0x2100 - 0x2100: 0x1100, 0x2101: 0x1100, 0x2102: 0x1100, 0x2103: 0x1100, 0x2104: 0x1100, 0x2105: 0x1100, - 0x2106: 0x1100, 0x2107: 0x1100, 0x2108: 0x1100, 0x2109: 0x1100, 0x210a: 0x1100, 0x210b: 0x1100, - 0x210c: 0x1100, 0x210d: 0x1100, 0x210e: 0x1100, 0x210f: 0x1100, 0x2110: 0x1100, 0x2111: 0x1100, - 0x2112: 0x1100, 0x2113: 0x1100, 0x2114: 0x1100, 0x2115: 0x1100, 0x2116: 0x1100, 0x2117: 0x1100, - 0x2118: 0x1100, 0x2119: 0x1100, 0x211a: 0x1100, 0x211b: 0x1100, 0x211c: 0x1100, 0x211d: 0x1100, - 0x211e: 0x1100, 0x211f: 0x1100, 0x2120: 0x1100, 0x2121: 0x1100, 0x2122: 0x1100, 0x2123: 0x1100, - 0x2124: 0x1100, 0x2125: 0x1100, 0x2126: 0x1100, 0x2127: 0x1100, 0x2128: 0x1100, 0x2129: 0x1100, - 0x212a: 0x1100, 0x212b: 0x1100, 0x212c: 0x1100, 0x212d: 0x1100, 0x212e: 0x1100, 0x212f: 0x1100, - 0x2130: 0x1100, 0x2131: 0x1100, 0x2132: 0x1100, 0x2133: 0x1100, 0x2134: 0x1100, 0x2135: 0x1100, - 0x2136: 0x1100, 0x2137: 0x1100, 0x2138: 0x1100, 0x2139: 0x1100, 0x213a: 0x1100, 0x213b: 0x1100, - 0x213c: 0x1100, 0x213d: 0x1100, 0x213e: 0x1100, 0x213f: 0x1100, + 0x212d: 0x0009, // Block 0x85, offset 0x2140 0x2140: 0x1100, 0x2141: 0x1100, 0x2142: 0x1100, 0x2143: 0x1100, 0x2144: 0x1100, 0x2145: 0x1100, 0x2146: 0x1100, 0x2147: 0x1100, 0x2148: 0x1100, 0x2149: 0x1100, 0x214a: 0x1100, 0x214b: 0x1100, @@ -6161,38 +6162,38 @@ var charInfoValues = [10944]uint16{ 0x2152: 0x1100, 0x2153: 0x1100, 0x2154: 0x1100, 0x2155: 0x1100, 0x2156: 0x1100, 0x2157: 0x1100, 0x2158: 0x1100, 0x2159: 0x1100, 0x215a: 0x1100, 0x215b: 0x1100, 0x215c: 0x1100, 0x215d: 0x1100, 0x215e: 0x1100, 0x215f: 0x1100, 0x2160: 0x1100, 0x2161: 0x1100, 0x2162: 0x1100, 0x2163: 0x1100, + 0x2164: 0x1100, 0x2165: 0x1100, 0x2166: 0x1100, 0x2167: 0x1100, 0x2168: 0x1100, 0x2169: 0x1100, + 0x216a: 0x1100, 0x216b: 0x1100, 0x216c: 0x1100, 0x216d: 0x1100, 0x216e: 0x1100, 0x216f: 0x1100, + 0x2170: 0x1100, 0x2171: 0x1100, 0x2172: 0x1100, 0x2173: 0x1100, 0x2174: 0x1100, 0x2175: 0x1100, + 0x2176: 0x1100, 0x2177: 0x1100, 0x2178: 0x1100, 0x2179: 0x1100, 0x217a: 0x1100, 0x217b: 0x1100, + 0x217c: 0x1100, 0x217d: 0x1100, 0x217e: 0x1100, 0x217f: 0x1100, // Block 0x86, offset 0x2180 - 0x2180: 0x3300, 0x2181: 0x3300, 0x2182: 0x3300, 0x2183: 0x3300, 0x2184: 0x3300, 0x2185: 0x3300, - 0x2186: 0x3300, 0x2187: 0x3300, 0x2188: 0x3300, 0x2189: 0x3300, 0x218a: 0x3300, 0x218b: 0x3300, - 0x218c: 0x3300, 0x218d: 0x3300, 0x218e: 0x3300, 0x218f: 0x3300, 0x2190: 0x3300, 0x2191: 0x3300, - 0x2192: 0x3300, 0x2193: 0x3300, 0x2194: 0x3300, 0x2195: 0x3300, 0x2196: 0x3300, 0x2197: 0x3300, - 0x2198: 0x3300, 0x2199: 0x3300, 0x219a: 0x3300, 0x219b: 0x3300, 0x219c: 0x3300, 0x219d: 0x3300, - 0x219e: 0x3300, 0x219f: 0x3300, 0x21a0: 0x3300, 0x21a1: 0x3300, 0x21a2: 0x3300, 0x21a3: 0x3300, - 0x21a4: 0x3300, 0x21a5: 0x3300, 0x21a6: 0x3300, 0x21a7: 0x3300, 0x21a8: 0x3300, 0x21a9: 0x3300, - 0x21aa: 0x3300, 0x21ab: 0x3300, 0x21ac: 0x3300, 0x21ad: 0x3300, 0x21ae: 0x3300, 0x21af: 0x3300, - 0x21b0: 0x3300, 0x21b1: 0x3300, 0x21b2: 0x3300, 0x21b3: 0x3300, 0x21b4: 0x3300, 0x21b5: 0x3300, - 0x21b6: 0x3300, 0x21b7: 0x3300, 0x21b8: 0x3300, 0x21b9: 0x3300, 0x21ba: 0x3300, 0x21bb: 0x3300, - 0x21bc: 0x3300, 0x21bd: 0x3300, 0x21be: 0x3300, 0x21bf: 0x3300, + 0x2180: 0x1100, 0x2181: 0x1100, 0x2182: 0x1100, 0x2183: 0x1100, 0x2184: 0x1100, 0x2185: 0x1100, + 0x2186: 0x1100, 0x2187: 0x1100, 0x2188: 0x1100, 0x2189: 0x1100, 0x218a: 0x1100, 0x218b: 0x1100, + 0x218c: 0x1100, 0x218d: 0x1100, 0x218e: 0x1100, 0x218f: 0x1100, 0x2190: 0x1100, 0x2191: 0x1100, + 0x2192: 0x1100, 0x2193: 0x1100, 0x2194: 0x1100, 0x2195: 0x1100, 0x2196: 0x1100, 0x2197: 0x1100, + 0x2198: 0x1100, 0x2199: 0x1100, 0x219a: 0x1100, 0x219b: 0x1100, 0x219c: 0x1100, 0x219d: 0x1100, + 0x219e: 0x1100, 0x219f: 0x1100, 0x21a0: 0x1100, 0x21a1: 0x1100, 0x21a2: 0x1100, 0x21a3: 0x1100, // Block 0x87, offset 0x21c0 0x21c0: 0x3300, 0x21c1: 0x3300, 0x21c2: 0x3300, 0x21c3: 0x3300, 0x21c4: 0x3300, 0x21c5: 0x3300, 0x21c6: 0x3300, 0x21c7: 0x3300, 0x21c8: 0x3300, 0x21c9: 0x3300, 0x21ca: 0x3300, 0x21cb: 0x3300, - 0x21cc: 0x3300, 0x21cd: 0x3300, 0x21d0: 0x3300, - 0x21d2: 0x3300, 0x21d5: 0x3300, 0x21d6: 0x3300, 0x21d7: 0x3300, + 0x21cc: 0x3300, 0x21cd: 0x3300, 0x21ce: 0x3300, 0x21cf: 0x3300, 0x21d0: 0x3300, 0x21d1: 0x3300, + 0x21d2: 0x3300, 0x21d3: 0x3300, 0x21d4: 0x3300, 0x21d5: 0x3300, 0x21d6: 0x3300, 0x21d7: 0x3300, 0x21d8: 0x3300, 0x21d9: 0x3300, 0x21da: 0x3300, 0x21db: 0x3300, 0x21dc: 0x3300, 0x21dd: 0x3300, - 0x21de: 0x3300, 0x21e0: 0x3300, 0x21e2: 0x3300, - 0x21e5: 0x3300, 0x21e6: 0x3300, - 0x21ea: 0x3300, 0x21eb: 0x3300, 0x21ec: 0x3300, 0x21ed: 0x3300, + 0x21de: 0x3300, 0x21df: 0x3300, 0x21e0: 0x3300, 0x21e1: 0x3300, 0x21e2: 0x3300, 0x21e3: 0x3300, + 0x21e4: 0x3300, 0x21e5: 0x3300, 0x21e6: 0x3300, 0x21e7: 0x3300, 0x21e8: 0x3300, 0x21e9: 0x3300, + 0x21ea: 0x3300, 0x21eb: 0x3300, 0x21ec: 0x3300, 0x21ed: 0x3300, 0x21ee: 0x3300, 0x21ef: 0x3300, 0x21f0: 0x3300, 0x21f1: 0x3300, 0x21f2: 0x3300, 0x21f3: 0x3300, 0x21f4: 0x3300, 0x21f5: 0x3300, 0x21f6: 0x3300, 0x21f7: 0x3300, 0x21f8: 0x3300, 0x21f9: 0x3300, 0x21fa: 0x3300, 0x21fb: 0x3300, 0x21fc: 0x3300, 0x21fd: 0x3300, 0x21fe: 0x3300, 0x21ff: 0x3300, // Block 0x88, offset 0x2200 0x2200: 0x3300, 0x2201: 0x3300, 0x2202: 0x3300, 0x2203: 0x3300, 0x2204: 0x3300, 0x2205: 0x3300, 0x2206: 0x3300, 0x2207: 0x3300, 0x2208: 0x3300, 0x2209: 0x3300, 0x220a: 0x3300, 0x220b: 0x3300, - 0x220c: 0x3300, 0x220d: 0x3300, 0x220e: 0x3300, 0x220f: 0x3300, 0x2210: 0x3300, 0x2211: 0x3300, - 0x2212: 0x3300, 0x2213: 0x3300, 0x2214: 0x3300, 0x2215: 0x3300, 0x2216: 0x3300, 0x2217: 0x3300, + 0x220c: 0x3300, 0x220d: 0x3300, 0x2210: 0x3300, + 0x2212: 0x3300, 0x2215: 0x3300, 0x2216: 0x3300, 0x2217: 0x3300, 0x2218: 0x3300, 0x2219: 0x3300, 0x221a: 0x3300, 0x221b: 0x3300, 0x221c: 0x3300, 0x221d: 0x3300, - 0x221e: 0x3300, 0x221f: 0x3300, 0x2220: 0x3300, 0x2221: 0x3300, 0x2222: 0x3300, 0x2223: 0x3300, - 0x2224: 0x3300, 0x2225: 0x3300, 0x2226: 0x3300, 0x2227: 0x3300, 0x2228: 0x3300, 0x2229: 0x3300, + 0x221e: 0x3300, 0x2220: 0x3300, 0x2222: 0x3300, + 0x2225: 0x3300, 0x2226: 0x3300, 0x222a: 0x3300, 0x222b: 0x3300, 0x222c: 0x3300, 0x222d: 0x3300, 0x2230: 0x3300, 0x2231: 0x3300, 0x2232: 0x3300, 0x2233: 0x3300, 0x2234: 0x3300, 0x2235: 0x3300, 0x2236: 0x3300, 0x2237: 0x3300, 0x2238: 0x3300, 0x2239: 0x3300, 0x223a: 0x3300, 0x223b: 0x3300, @@ -6202,63 +6203,65 @@ var charInfoValues = [10944]uint16{ 0x2246: 0x3300, 0x2247: 0x3300, 0x2248: 0x3300, 0x2249: 0x3300, 0x224a: 0x3300, 0x224b: 0x3300, 0x224c: 0x3300, 0x224d: 0x3300, 0x224e: 0x3300, 0x224f: 0x3300, 0x2250: 0x3300, 0x2251: 0x3300, 0x2252: 0x3300, 0x2253: 0x3300, 0x2254: 0x3300, 0x2255: 0x3300, 0x2256: 0x3300, 0x2257: 0x3300, - 0x2258: 0x3300, 0x2259: 0x3300, + 0x2258: 0x3300, 0x2259: 0x3300, 0x225a: 0x3300, 0x225b: 0x3300, 0x225c: 0x3300, 0x225d: 0x3300, + 0x225e: 0x3300, 0x225f: 0x3300, 0x2260: 0x3300, 0x2261: 0x3300, 0x2262: 0x3300, 0x2263: 0x3300, + 0x2264: 0x3300, 0x2265: 0x3300, 0x2266: 0x3300, 0x2267: 0x3300, 0x2268: 0x3300, 0x2269: 0x3300, + 0x226a: 0x3300, 0x226b: 0x3300, 0x226c: 0x3300, 0x226d: 0x3300, + 0x2270: 0x3300, 0x2271: 0x3300, 0x2272: 0x3300, 0x2273: 0x3300, 0x2274: 0x3300, 0x2275: 0x3300, + 0x2276: 0x3300, 0x2277: 0x3300, 0x2278: 0x3300, 0x2279: 0x3300, 0x227a: 0x3300, 0x227b: 0x3300, + 0x227c: 0x3300, 0x227d: 0x3300, 0x227e: 0x3300, 0x227f: 0x3300, // Block 0x8a, offset 0x2280 - 0x2280: 0x3000, 0x2281: 0x3000, 0x2282: 0x3000, 0x2283: 0x3000, 0x2284: 0x3000, 0x2285: 0x3000, - 0x2286: 0x3000, - 0x2293: 0x3000, 0x2294: 0x3000, 0x2295: 0x3000, 0x2296: 0x3000, 0x2297: 0x3000, - 0x229d: 0x3300, - 0x229e: 0x001a, 0x229f: 0x3300, 0x22a0: 0x3000, 0x22a1: 0x3000, 0x22a2: 0x3000, 0x22a3: 0x3000, - 0x22a4: 0x3000, 0x22a5: 0x3000, 0x22a6: 0x3000, 0x22a7: 0x3000, 0x22a8: 0x3000, 0x22a9: 0x3000, - 0x22aa: 0x3300, 0x22ab: 0x3300, 0x22ac: 0x3300, 0x22ad: 0x3300, 0x22ae: 0x3300, 0x22af: 0x3300, - 0x22b0: 0x3300, 0x22b1: 0x3300, 0x22b2: 0x3300, 0x22b3: 0x3300, 0x22b4: 0x3300, 0x22b5: 0x3300, - 0x22b6: 0x3300, 0x22b8: 0x3300, 0x22b9: 0x3300, 0x22ba: 0x3300, 0x22bb: 0x3300, - 0x22bc: 0x3300, 0x22be: 0x3300, + 0x2280: 0x3300, 0x2281: 0x3300, 0x2282: 0x3300, 0x2283: 0x3300, 0x2284: 0x3300, 0x2285: 0x3300, + 0x2286: 0x3300, 0x2287: 0x3300, 0x2288: 0x3300, 0x2289: 0x3300, 0x228a: 0x3300, 0x228b: 0x3300, + 0x228c: 0x3300, 0x228d: 0x3300, 0x228e: 0x3300, 0x228f: 0x3300, 0x2290: 0x3300, 0x2291: 0x3300, + 0x2292: 0x3300, 0x2293: 0x3300, 0x2294: 0x3300, 0x2295: 0x3300, 0x2296: 0x3300, 0x2297: 0x3300, + 0x2298: 0x3300, 0x2299: 0x3300, // Block 0x8b, offset 0x22c0 - 0x22c0: 0x3300, 0x22c1: 0x3300, 0x22c3: 0x3300, 0x22c4: 0x3300, - 0x22c6: 0x3300, 0x22c7: 0x3300, 0x22c8: 0x3300, 0x22c9: 0x3300, 0x22ca: 0x3300, 0x22cb: 0x3300, - 0x22cc: 0x3300, 0x22cd: 0x3300, 0x22ce: 0x3300, 0x22cf: 0x3000, 0x22d0: 0x3000, 0x22d1: 0x3000, - 0x22d2: 0x3000, 0x22d3: 0x3000, 0x22d4: 0x3000, 0x22d5: 0x3000, 0x22d6: 0x3000, 0x22d7: 0x3000, - 0x22d8: 0x3000, 0x22d9: 0x3000, 0x22da: 0x3000, 0x22db: 0x3000, 0x22dc: 0x3000, 0x22dd: 0x3000, - 0x22de: 0x3000, 0x22df: 0x3000, 0x22e0: 0x3000, 0x22e1: 0x3000, 0x22e2: 0x3000, 0x22e3: 0x3000, + 0x22c0: 0x3000, 0x22c1: 0x3000, 0x22c2: 0x3000, 0x22c3: 0x3000, 0x22c4: 0x3000, 0x22c5: 0x3000, + 0x22c6: 0x3000, + 0x22d3: 0x3000, 0x22d4: 0x3000, 0x22d5: 0x3000, 0x22d6: 0x3000, 0x22d7: 0x3000, + 0x22dd: 0x3300, + 0x22de: 0x001a, 0x22df: 0x3300, 0x22e0: 0x3000, 0x22e1: 0x3000, 0x22e2: 0x3000, 0x22e3: 0x3000, 0x22e4: 0x3000, 0x22e5: 0x3000, 0x22e6: 0x3000, 0x22e7: 0x3000, 0x22e8: 0x3000, 0x22e9: 0x3000, - 0x22ea: 0x3000, 0x22eb: 0x3000, 0x22ec: 0x3000, 0x22ed: 0x3000, 0x22ee: 0x3000, 0x22ef: 0x3000, - 0x22f0: 0x3000, 0x22f1: 0x3000, 0x22f2: 0x3000, 0x22f3: 0x3000, 0x22f4: 0x3000, 0x22f5: 0x3000, - 0x22f6: 0x3000, 0x22f7: 0x3000, 0x22f8: 0x3000, 0x22f9: 0x3000, 0x22fa: 0x3000, 0x22fb: 0x3000, - 0x22fc: 0x3000, 0x22fd: 0x3000, 0x22fe: 0x3000, 0x22ff: 0x3000, + 0x22ea: 0x3300, 0x22eb: 0x3300, 0x22ec: 0x3300, 0x22ed: 0x3300, 0x22ee: 0x3300, 0x22ef: 0x3300, + 0x22f0: 0x3300, 0x22f1: 0x3300, 0x22f2: 0x3300, 0x22f3: 0x3300, 0x22f4: 0x3300, 0x22f5: 0x3300, + 0x22f6: 0x3300, 0x22f8: 0x3300, 0x22f9: 0x3300, 0x22fa: 0x3300, 0x22fb: 0x3300, + 0x22fc: 0x3300, 0x22fe: 0x3300, // Block 0x8c, offset 0x2300 - 0x2300: 0x3000, 0x2301: 0x3000, 0x2302: 0x3000, 0x2303: 0x3000, 0x2304: 0x3000, 0x2305: 0x3000, - 0x2306: 0x3000, 0x2307: 0x3000, 0x2308: 0x3000, 0x2309: 0x3000, 0x230a: 0x3000, 0x230b: 0x3000, - 0x230c: 0x3000, 0x230d: 0x3000, 0x230e: 0x3000, 0x230f: 0x3000, 0x2310: 0x3000, 0x2311: 0x3000, + 0x2300: 0x3300, 0x2301: 0x3300, 0x2303: 0x3300, 0x2304: 0x3300, + 0x2306: 0x3300, 0x2307: 0x3300, 0x2308: 0x3300, 0x2309: 0x3300, 0x230a: 0x3300, 0x230b: 0x3300, + 0x230c: 0x3300, 0x230d: 0x3300, 0x230e: 0x3300, 0x230f: 0x3000, 0x2310: 0x3000, 0x2311: 0x3000, 0x2312: 0x3000, 0x2313: 0x3000, 0x2314: 0x3000, 0x2315: 0x3000, 0x2316: 0x3000, 0x2317: 0x3000, 0x2318: 0x3000, 0x2319: 0x3000, 0x231a: 0x3000, 0x231b: 0x3000, 0x231c: 0x3000, 0x231d: 0x3000, 0x231e: 0x3000, 0x231f: 0x3000, 0x2320: 0x3000, 0x2321: 0x3000, 0x2322: 0x3000, 0x2323: 0x3000, 0x2324: 0x3000, 0x2325: 0x3000, 0x2326: 0x3000, 0x2327: 0x3000, 0x2328: 0x3000, 0x2329: 0x3000, 0x232a: 0x3000, 0x232b: 0x3000, 0x232c: 0x3000, 0x232d: 0x3000, 0x232e: 0x3000, 0x232f: 0x3000, - 0x2330: 0x3000, 0x2331: 0x3000, + 0x2330: 0x3000, 0x2331: 0x3000, 0x2332: 0x3000, 0x2333: 0x3000, 0x2334: 0x3000, 0x2335: 0x3000, + 0x2336: 0x3000, 0x2337: 0x3000, 0x2338: 0x3000, 0x2339: 0x3000, 0x233a: 0x3000, 0x233b: 0x3000, + 0x233c: 0x3000, 0x233d: 0x3000, 0x233e: 0x3000, 0x233f: 0x3000, // Block 0x8d, offset 0x2340 - 0x2353: 0x3000, 0x2354: 0x3000, 0x2355: 0x3000, 0x2356: 0x3000, 0x2357: 0x3000, + 0x2340: 0x3000, 0x2341: 0x3000, 0x2342: 0x3000, 0x2343: 0x3000, 0x2344: 0x3000, 0x2345: 0x3000, + 0x2346: 0x3000, 0x2347: 0x3000, 0x2348: 0x3000, 0x2349: 0x3000, 0x234a: 0x3000, 0x234b: 0x3000, + 0x234c: 0x3000, 0x234d: 0x3000, 0x234e: 0x3000, 0x234f: 0x3000, 0x2350: 0x3000, 0x2351: 0x3000, + 0x2352: 0x3000, 0x2353: 0x3000, 0x2354: 0x3000, 0x2355: 0x3000, 0x2356: 0x3000, 0x2357: 0x3000, 0x2358: 0x3000, 0x2359: 0x3000, 0x235a: 0x3000, 0x235b: 0x3000, 0x235c: 0x3000, 0x235d: 0x3000, 0x235e: 0x3000, 0x235f: 0x3000, 0x2360: 0x3000, 0x2361: 0x3000, 0x2362: 0x3000, 0x2363: 0x3000, 0x2364: 0x3000, 0x2365: 0x3000, 0x2366: 0x3000, 0x2367: 0x3000, 0x2368: 0x3000, 0x2369: 0x3000, 0x236a: 0x3000, 0x236b: 0x3000, 0x236c: 0x3000, 0x236d: 0x3000, 0x236e: 0x3000, 0x236f: 0x3000, - 0x2370: 0x3000, 0x2371: 0x3000, 0x2372: 0x3000, 0x2373: 0x3000, 0x2374: 0x3000, 0x2375: 0x3000, - 0x2376: 0x3000, 0x2377: 0x3000, 0x2378: 0x3000, 0x2379: 0x3000, 0x237a: 0x3000, 0x237b: 0x3000, - 0x237c: 0x3000, 0x237d: 0x3000, 0x237e: 0x3000, 0x237f: 0x3000, + 0x2370: 0x3000, 0x2371: 0x3000, // Block 0x8e, offset 0x2380 - 0x2380: 0x3000, 0x2381: 0x3000, 0x2382: 0x3000, 0x2383: 0x3000, 0x2384: 0x3000, 0x2385: 0x3000, - 0x2386: 0x3000, 0x2387: 0x3000, 0x2388: 0x3000, 0x2389: 0x3000, 0x238a: 0x3000, 0x238b: 0x3000, - 0x238c: 0x3000, 0x238d: 0x3000, 0x238e: 0x3000, 0x238f: 0x3000, 0x2390: 0x3000, 0x2391: 0x3000, - 0x2392: 0x3000, 0x2393: 0x3000, 0x2394: 0x3000, 0x2395: 0x3000, 0x2396: 0x3000, 0x2397: 0x3000, + 0x2393: 0x3000, 0x2394: 0x3000, 0x2395: 0x3000, 0x2396: 0x3000, 0x2397: 0x3000, 0x2398: 0x3000, 0x2399: 0x3000, 0x239a: 0x3000, 0x239b: 0x3000, 0x239c: 0x3000, 0x239d: 0x3000, 0x239e: 0x3000, 0x239f: 0x3000, 0x23a0: 0x3000, 0x23a1: 0x3000, 0x23a2: 0x3000, 0x23a3: 0x3000, 0x23a4: 0x3000, 0x23a5: 0x3000, 0x23a6: 0x3000, 0x23a7: 0x3000, 0x23a8: 0x3000, 0x23a9: 0x3000, 0x23aa: 0x3000, 0x23ab: 0x3000, 0x23ac: 0x3000, 0x23ad: 0x3000, 0x23ae: 0x3000, 0x23af: 0x3000, 0x23b0: 0x3000, 0x23b1: 0x3000, 0x23b2: 0x3000, 0x23b3: 0x3000, 0x23b4: 0x3000, 0x23b5: 0x3000, 0x23b6: 0x3000, 0x23b7: 0x3000, 0x23b8: 0x3000, 0x23b9: 0x3000, 0x23ba: 0x3000, 0x23bb: 0x3000, - 0x23bc: 0x3000, 0x23bd: 0x3000, + 0x23bc: 0x3000, 0x23bd: 0x3000, 0x23be: 0x3000, 0x23bf: 0x3000, // Block 0x8f, offset 0x23c0 - 0x23d0: 0x3000, 0x23d1: 0x3000, + 0x23c0: 0x3000, 0x23c1: 0x3000, 0x23c2: 0x3000, 0x23c3: 0x3000, 0x23c4: 0x3000, 0x23c5: 0x3000, + 0x23c6: 0x3000, 0x23c7: 0x3000, 0x23c8: 0x3000, 0x23c9: 0x3000, 0x23ca: 0x3000, 0x23cb: 0x3000, + 0x23cc: 0x3000, 0x23cd: 0x3000, 0x23ce: 0x3000, 0x23cf: 0x3000, 0x23d0: 0x3000, 0x23d1: 0x3000, 0x23d2: 0x3000, 0x23d3: 0x3000, 0x23d4: 0x3000, 0x23d5: 0x3000, 0x23d6: 0x3000, 0x23d7: 0x3000, 0x23d8: 0x3000, 0x23d9: 0x3000, 0x23da: 0x3000, 0x23db: 0x3000, 0x23dc: 0x3000, 0x23dd: 0x3000, 0x23de: 0x3000, 0x23df: 0x3000, 0x23e0: 0x3000, 0x23e1: 0x3000, 0x23e2: 0x3000, 0x23e3: 0x3000, @@ -6266,11 +6269,9 @@ var charInfoValues = [10944]uint16{ 0x23ea: 0x3000, 0x23eb: 0x3000, 0x23ec: 0x3000, 0x23ed: 0x3000, 0x23ee: 0x3000, 0x23ef: 0x3000, 0x23f0: 0x3000, 0x23f1: 0x3000, 0x23f2: 0x3000, 0x23f3: 0x3000, 0x23f4: 0x3000, 0x23f5: 0x3000, 0x23f6: 0x3000, 0x23f7: 0x3000, 0x23f8: 0x3000, 0x23f9: 0x3000, 0x23fa: 0x3000, 0x23fb: 0x3000, - 0x23fc: 0x3000, 0x23fd: 0x3000, 0x23fe: 0x3000, 0x23ff: 0x3000, + 0x23fc: 0x3000, 0x23fd: 0x3000, // Block 0x90, offset 0x2400 - 0x2400: 0x3000, 0x2401: 0x3000, 0x2402: 0x3000, 0x2403: 0x3000, 0x2404: 0x3000, 0x2405: 0x3000, - 0x2406: 0x3000, 0x2407: 0x3000, 0x2408: 0x3000, 0x2409: 0x3000, 0x240a: 0x3000, 0x240b: 0x3000, - 0x240c: 0x3000, 0x240d: 0x3000, 0x240e: 0x3000, 0x240f: 0x3000, + 0x2410: 0x3000, 0x2411: 0x3000, 0x2412: 0x3000, 0x2413: 0x3000, 0x2414: 0x3000, 0x2415: 0x3000, 0x2416: 0x3000, 0x2417: 0x3000, 0x2418: 0x3000, 0x2419: 0x3000, 0x241a: 0x3000, 0x241b: 0x3000, 0x241c: 0x3000, 0x241d: 0x3000, 0x241e: 0x3000, 0x241f: 0x3000, 0x2420: 0x3000, 0x2421: 0x3000, 0x2422: 0x3000, 0x2423: 0x3000, @@ -6281,45 +6282,45 @@ var charInfoValues = [10944]uint16{ 0x243c: 0x3000, 0x243d: 0x3000, 0x243e: 0x3000, 0x243f: 0x3000, // Block 0x91, offset 0x2440 0x2440: 0x3000, 0x2441: 0x3000, 0x2442: 0x3000, 0x2443: 0x3000, 0x2444: 0x3000, 0x2445: 0x3000, - 0x2446: 0x3000, 0x2447: 0x3000, + 0x2446: 0x3000, 0x2447: 0x3000, 0x2448: 0x3000, 0x2449: 0x3000, 0x244a: 0x3000, 0x244b: 0x3000, + 0x244c: 0x3000, 0x244d: 0x3000, 0x244e: 0x3000, 0x244f: 0x3000, + 0x2452: 0x3000, 0x2453: 0x3000, 0x2454: 0x3000, 0x2455: 0x3000, 0x2456: 0x3000, 0x2457: 0x3000, + 0x2458: 0x3000, 0x2459: 0x3000, 0x245a: 0x3000, 0x245b: 0x3000, 0x245c: 0x3000, 0x245d: 0x3000, + 0x245e: 0x3000, 0x245f: 0x3000, 0x2460: 0x3000, 0x2461: 0x3000, 0x2462: 0x3000, 0x2463: 0x3000, + 0x2464: 0x3000, 0x2465: 0x3000, 0x2466: 0x3000, 0x2467: 0x3000, 0x2468: 0x3000, 0x2469: 0x3000, + 0x246a: 0x3000, 0x246b: 0x3000, 0x246c: 0x3000, 0x246d: 0x3000, 0x246e: 0x3000, 0x246f: 0x3000, 0x2470: 0x3000, 0x2471: 0x3000, 0x2472: 0x3000, 0x2473: 0x3000, 0x2474: 0x3000, 0x2475: 0x3000, 0x2476: 0x3000, 0x2477: 0x3000, 0x2478: 0x3000, 0x2479: 0x3000, 0x247a: 0x3000, 0x247b: 0x3000, - 0x247c: 0x3000, + 0x247c: 0x3000, 0x247d: 0x3000, 0x247e: 0x3000, 0x247f: 0x3000, // Block 0x92, offset 0x2480 - 0x2490: 0x3000, 0x2491: 0x3000, - 0x2492: 0x3000, 0x2493: 0x3000, 0x2494: 0x3000, 0x2495: 0x3000, 0x2496: 0x3000, 0x2497: 0x3000, - 0x2498: 0x3000, 0x2499: 0x3000, - 0x24a0: 0x00e6, 0x24a1: 0x00e6, 0x24a2: 0x00e6, 0x24a3: 0x00e6, - 0x24a4: 0x00e6, 0x24a5: 0x00e6, 0x24a6: 0x00e6, + 0x2480: 0x3000, 0x2481: 0x3000, 0x2482: 0x3000, 0x2483: 0x3000, 0x2484: 0x3000, 0x2485: 0x3000, + 0x2486: 0x3000, 0x2487: 0x3000, 0x24b0: 0x3000, 0x24b1: 0x3000, 0x24b2: 0x3000, 0x24b3: 0x3000, 0x24b4: 0x3000, 0x24b5: 0x3000, 0x24b6: 0x3000, 0x24b7: 0x3000, 0x24b8: 0x3000, 0x24b9: 0x3000, 0x24ba: 0x3000, 0x24bb: 0x3000, - 0x24bc: 0x3000, 0x24bd: 0x3000, 0x24be: 0x3000, 0x24bf: 0x3000, + 0x24bc: 0x3000, // Block 0x93, offset 0x24c0 - 0x24c0: 0x3000, 0x24c1: 0x3000, 0x24c2: 0x3000, 0x24c3: 0x3000, 0x24c4: 0x3000, - 0x24c7: 0x3000, 0x24c8: 0x3000, 0x24c9: 0x3000, 0x24ca: 0x3000, 0x24cb: 0x3000, - 0x24cc: 0x3000, 0x24cd: 0x3000, 0x24ce: 0x3000, 0x24cf: 0x3000, 0x24d0: 0x3000, 0x24d1: 0x3000, - 0x24d2: 0x3000, 0x24d4: 0x3000, 0x24d5: 0x3000, 0x24d6: 0x3000, 0x24d7: 0x3000, - 0x24d8: 0x3000, 0x24d9: 0x3000, 0x24da: 0x3000, 0x24db: 0x3000, 0x24dc: 0x3000, 0x24dd: 0x3000, - 0x24de: 0x3000, 0x24df: 0x3000, 0x24e0: 0x3000, 0x24e1: 0x3000, 0x24e2: 0x3000, 0x24e3: 0x3000, - 0x24e4: 0x3000, 0x24e5: 0x3000, 0x24e6: 0x3000, 0x24e8: 0x3000, 0x24e9: 0x3000, - 0x24ea: 0x3000, 0x24eb: 0x3000, - 0x24f0: 0x3000, 0x24f1: 0x3000, 0x24f2: 0x3000, 0x24f4: 0x3000, + 0x24d0: 0x3000, 0x24d1: 0x3000, + 0x24d2: 0x3000, 0x24d3: 0x3000, 0x24d4: 0x3000, 0x24d5: 0x3000, 0x24d6: 0x3000, 0x24d7: 0x3000, + 0x24d8: 0x3000, 0x24d9: 0x3000, + 0x24e0: 0x00e6, 0x24e1: 0x00e6, 0x24e2: 0x00e6, 0x24e3: 0x00e6, + 0x24e4: 0x00e6, 0x24e5: 0x00e6, 0x24e6: 0x00e6, + 0x24f0: 0x3000, 0x24f1: 0x3000, 0x24f2: 0x3000, 0x24f3: 0x3000, 0x24f4: 0x3000, 0x24f5: 0x3000, 0x24f6: 0x3000, 0x24f7: 0x3000, 0x24f8: 0x3000, 0x24f9: 0x3000, 0x24fa: 0x3000, 0x24fb: 0x3000, 0x24fc: 0x3000, 0x24fd: 0x3000, 0x24fe: 0x3000, 0x24ff: 0x3000, // Block 0x94, offset 0x2500 - 0x2500: 0x3000, 0x2501: 0x3000, 0x2502: 0x3000, 0x2503: 0x3000, 0x2504: 0x3000, 0x2505: 0x3000, - 0x2506: 0x3000, 0x2507: 0x3000, 0x2508: 0x3000, 0x2509: 0x3000, 0x250a: 0x3000, 0x250b: 0x3000, + 0x2500: 0x3000, 0x2501: 0x3000, 0x2502: 0x3000, 0x2503: 0x3000, 0x2504: 0x3000, + 0x2507: 0x3000, 0x2508: 0x3000, 0x2509: 0x3000, 0x250a: 0x3000, 0x250b: 0x3000, 0x250c: 0x3000, 0x250d: 0x3000, 0x250e: 0x3000, 0x250f: 0x3000, 0x2510: 0x3000, 0x2511: 0x3000, - 0x2512: 0x3000, 0x2513: 0x3000, 0x2514: 0x3000, 0x2515: 0x3000, 0x2516: 0x3000, 0x2517: 0x3000, + 0x2512: 0x3000, 0x2514: 0x3000, 0x2515: 0x3000, 0x2516: 0x3000, 0x2517: 0x3000, 0x2518: 0x3000, 0x2519: 0x3000, 0x251a: 0x3000, 0x251b: 0x3000, 0x251c: 0x3000, 0x251d: 0x3000, 0x251e: 0x3000, 0x251f: 0x3000, 0x2520: 0x3000, 0x2521: 0x3000, 0x2522: 0x3000, 0x2523: 0x3000, - 0x2524: 0x3000, 0x2525: 0x3000, 0x2526: 0x3000, 0x2527: 0x3000, 0x2528: 0x3000, 0x2529: 0x3000, - 0x252a: 0x3000, 0x252b: 0x3000, 0x252c: 0x3000, 0x252d: 0x3000, 0x252e: 0x3000, 0x252f: 0x3000, - 0x2530: 0x3000, 0x2531: 0x3000, 0x2532: 0x3000, 0x2533: 0x3000, 0x2534: 0x3000, 0x2535: 0x3000, + 0x2524: 0x3000, 0x2525: 0x3000, 0x2526: 0x3000, 0x2528: 0x3000, 0x2529: 0x3000, + 0x252a: 0x3000, 0x252b: 0x3000, + 0x2530: 0x3000, 0x2531: 0x3000, 0x2532: 0x3000, 0x2534: 0x3000, 0x2536: 0x3000, 0x2537: 0x3000, 0x2538: 0x3000, 0x2539: 0x3000, 0x253a: 0x3000, 0x253b: 0x3000, - 0x253c: 0x3000, + 0x253c: 0x3000, 0x253d: 0x3000, 0x253e: 0x3000, 0x253f: 0x3000, // Block 0x95, offset 0x2540 - 0x2541: 0x3000, 0x2542: 0x3000, 0x2543: 0x3000, 0x2544: 0x3000, 0x2545: 0x3000, + 0x2540: 0x3000, 0x2541: 0x3000, 0x2542: 0x3000, 0x2543: 0x3000, 0x2544: 0x3000, 0x2545: 0x3000, 0x2546: 0x3000, 0x2547: 0x3000, 0x2548: 0x3000, 0x2549: 0x3000, 0x254a: 0x3000, 0x254b: 0x3000, 0x254c: 0x3000, 0x254d: 0x3000, 0x254e: 0x3000, 0x254f: 0x3000, 0x2550: 0x3000, 0x2551: 0x3000, 0x2552: 0x3000, 0x2553: 0x3000, 0x2554: 0x3000, 0x2555: 0x3000, 0x2556: 0x3000, 0x2557: 0x3000, @@ -6329,112 +6330,112 @@ var charInfoValues = [10944]uint16{ 0x256a: 0x3000, 0x256b: 0x3000, 0x256c: 0x3000, 0x256d: 0x3000, 0x256e: 0x3000, 0x256f: 0x3000, 0x2570: 0x3000, 0x2571: 0x3000, 0x2572: 0x3000, 0x2573: 0x3000, 0x2574: 0x3000, 0x2575: 0x3000, 0x2576: 0x3000, 0x2577: 0x3000, 0x2578: 0x3000, 0x2579: 0x3000, 0x257a: 0x3000, 0x257b: 0x3000, - 0x257c: 0x3000, 0x257d: 0x3000, 0x257e: 0x3000, 0x257f: 0x3000, + 0x257c: 0x3000, // Block 0x96, offset 0x2580 - 0x2582: 0x3000, 0x2583: 0x3000, 0x2584: 0x3000, 0x2585: 0x3000, - 0x2586: 0x3000, 0x2587: 0x3000, 0x258a: 0x3000, 0x258b: 0x3000, - 0x258c: 0x3000, 0x258d: 0x3000, 0x258e: 0x3000, 0x258f: 0x3000, + 0x2581: 0x3000, 0x2582: 0x3000, 0x2583: 0x3000, 0x2584: 0x3000, 0x2585: 0x3000, + 0x2586: 0x3000, 0x2587: 0x3000, 0x2588: 0x3000, 0x2589: 0x3000, 0x258a: 0x3000, 0x258b: 0x3000, + 0x258c: 0x3000, 0x258d: 0x3000, 0x258e: 0x3000, 0x258f: 0x3000, 0x2590: 0x3000, 0x2591: 0x3000, 0x2592: 0x3000, 0x2593: 0x3000, 0x2594: 0x3000, 0x2595: 0x3000, 0x2596: 0x3000, 0x2597: 0x3000, - 0x259a: 0x3000, 0x259b: 0x3000, 0x259c: 0x3000, - 0x25a0: 0x3000, 0x25a1: 0x3000, 0x25a2: 0x3000, 0x25a3: 0x3000, - 0x25a4: 0x3000, 0x25a5: 0x3000, 0x25a6: 0x3000, 0x25a8: 0x3000, 0x25a9: 0x3000, - 0x25aa: 0x3000, 0x25ab: 0x3000, 0x25ac: 0x3000, 0x25ad: 0x3000, 0x25ae: 0x3000, + 0x2598: 0x3000, 0x2599: 0x3000, 0x259a: 0x3000, 0x259b: 0x3000, 0x259c: 0x3000, 0x259d: 0x3000, + 0x259e: 0x3000, 0x259f: 0x3000, 0x25a0: 0x3000, 0x25a1: 0x3000, 0x25a2: 0x3000, 0x25a3: 0x3000, + 0x25a4: 0x3000, 0x25a5: 0x3000, 0x25a6: 0x3000, 0x25a7: 0x3000, 0x25a8: 0x3000, 0x25a9: 0x3000, + 0x25aa: 0x3000, 0x25ab: 0x3000, 0x25ac: 0x3000, 0x25ad: 0x3000, 0x25ae: 0x3000, 0x25af: 0x3000, + 0x25b0: 0x3000, 0x25b1: 0x3000, 0x25b2: 0x3000, 0x25b3: 0x3000, 0x25b4: 0x3000, 0x25b5: 0x3000, + 0x25b6: 0x3000, 0x25b7: 0x3000, 0x25b8: 0x3000, 0x25b9: 0x3000, 0x25ba: 0x3000, 0x25bb: 0x3000, + 0x25bc: 0x3000, 0x25bd: 0x3000, 0x25be: 0x3000, 0x25bf: 0x3000, // Block 0x97, offset 0x25c0 - 0x25fd: 0x00dc, + 0x25c2: 0x3000, 0x25c3: 0x3000, 0x25c4: 0x3000, 0x25c5: 0x3000, + 0x25c6: 0x3000, 0x25c7: 0x3000, 0x25ca: 0x3000, 0x25cb: 0x3000, + 0x25cc: 0x3000, 0x25cd: 0x3000, 0x25ce: 0x3000, 0x25cf: 0x3000, + 0x25d2: 0x3000, 0x25d3: 0x3000, 0x25d4: 0x3000, 0x25d5: 0x3000, 0x25d6: 0x3000, 0x25d7: 0x3000, + 0x25da: 0x3000, 0x25db: 0x3000, 0x25dc: 0x3000, + 0x25e0: 0x3000, 0x25e1: 0x3000, 0x25e2: 0x3000, 0x25e3: 0x3000, + 0x25e4: 0x3000, 0x25e5: 0x3000, 0x25e6: 0x3000, 0x25e8: 0x3000, 0x25e9: 0x3000, + 0x25ea: 0x3000, 0x25eb: 0x3000, 0x25ec: 0x3000, 0x25ed: 0x3000, 0x25ee: 0x3000, // Block 0x98, offset 0x2600 - 0x260d: 0x00dc, 0x260f: 0x00e6, - 0x2638: 0x00e6, 0x2639: 0x0001, 0x263a: 0x00dc, - 0x263f: 0x0009, + 0x263d: 0x00dc, // Block 0x99, offset 0x2640 - 0x2659: 0x8800, 0x265a: 0x1100, 0x265b: 0x8800, 0x265c: 0x1100, - 0x2665: 0x8800, - 0x266b: 0x1100, - 0x2679: 0x0009, 0x267a: 0x6607, + 0x264d: 0x00dc, 0x264f: 0x00e6, + 0x2678: 0x00e6, 0x2679: 0x0001, 0x267a: 0x00dc, + 0x267f: 0x0009, // Block 0x9a, offset 0x2680 - 0x269e: 0x3300, 0x269f: 0x3300, 0x26a0: 0x3300, 0x26a1: 0x3300, 0x26a2: 0x3300, 0x26a3: 0x3300, - 0x26a4: 0x3300, 0x26a5: 0x00d8, 0x26a6: 0x00d8, 0x26a7: 0x0001, 0x26a8: 0x0001, 0x26a9: 0x0001, - 0x26ad: 0x00e2, 0x26ae: 0x00d8, 0x26af: 0x00d8, - 0x26b0: 0x00d8, 0x26b1: 0x00d8, 0x26b2: 0x00d8, - 0x26bb: 0x00dc, - 0x26bc: 0x00dc, 0x26bd: 0x00dc, 0x26be: 0x00dc, 0x26bf: 0x00dc, + 0x2699: 0x8800, 0x269a: 0x1100, 0x269b: 0x8800, 0x269c: 0x1100, + 0x26a5: 0x8800, + 0x26ab: 0x1100, + 0x26b9: 0x0009, 0x26ba: 0x6607, // Block 0x9b, offset 0x26c0 - 0x26c0: 0x00dc, 0x26c1: 0x00dc, 0x26c2: 0x00dc, 0x26c5: 0x00e6, - 0x26c6: 0x00e6, 0x26c7: 0x00e6, 0x26c8: 0x00e6, 0x26c9: 0x00e6, 0x26ca: 0x00dc, 0x26cb: 0x00dc, - 0x26ea: 0x00e6, 0x26eb: 0x00e6, 0x26ec: 0x00e6, 0x26ed: 0x00e6, - 0x26fb: 0x3300, - 0x26fc: 0x3300, 0x26fd: 0x3300, 0x26fe: 0x3300, 0x26ff: 0x3300, + 0x26de: 0x3300, 0x26df: 0x3300, 0x26e0: 0x3300, 0x26e1: 0x3300, 0x26e2: 0x3300, 0x26e3: 0x3300, + 0x26e4: 0x3300, 0x26e5: 0x00d8, 0x26e6: 0x00d8, 0x26e7: 0x0001, 0x26e8: 0x0001, 0x26e9: 0x0001, + 0x26ed: 0x00e2, 0x26ee: 0x00d8, 0x26ef: 0x00d8, + 0x26f0: 0x00d8, 0x26f1: 0x00d8, 0x26f2: 0x00d8, + 0x26fb: 0x00dc, + 0x26fc: 0x00dc, 0x26fd: 0x00dc, 0x26fe: 0x00dc, 0x26ff: 0x00dc, // Block 0x9c, offset 0x2700 - 0x2700: 0x3300, + 0x2700: 0x00dc, 0x2701: 0x00dc, 0x2702: 0x00dc, 0x2705: 0x00e6, + 0x2706: 0x00e6, 0x2707: 0x00e6, 0x2708: 0x00e6, 0x2709: 0x00e6, 0x270a: 0x00dc, 0x270b: 0x00dc, + 0x272a: 0x00e6, 0x272b: 0x00e6, 0x272c: 0x00e6, 0x272d: 0x00e6, + 0x273b: 0x3300, + 0x273c: 0x3300, 0x273d: 0x3300, 0x273e: 0x3300, 0x273f: 0x3300, // Block 0x9d, offset 0x2740 - 0x2742: 0x00e6, 0x2743: 0x00e6, 0x2744: 0x00e6, + 0x2740: 0x3300, // Block 0x9e, offset 0x2780 - 0x2780: 0x3000, 0x2781: 0x3000, 0x2782: 0x3000, 0x2783: 0x3000, 0x2784: 0x3000, 0x2785: 0x3000, - 0x2786: 0x3000, 0x2787: 0x3000, 0x2788: 0x3000, 0x2789: 0x3000, 0x278a: 0x3000, 0x278b: 0x3000, - 0x278c: 0x3000, 0x278d: 0x3000, 0x278e: 0x3000, 0x278f: 0x3000, 0x2790: 0x3000, 0x2791: 0x3000, - 0x2792: 0x3000, 0x2793: 0x3000, 0x2794: 0x3000, 0x2796: 0x3000, 0x2797: 0x3000, - 0x2798: 0x3000, 0x2799: 0x3000, 0x279a: 0x3000, 0x279b: 0x3000, 0x279c: 0x3000, 0x279d: 0x3000, - 0x279e: 0x3000, 0x279f: 0x3000, 0x27a0: 0x3000, 0x27a1: 0x3000, 0x27a2: 0x3000, 0x27a3: 0x3000, - 0x27a4: 0x3000, 0x27a5: 0x3000, 0x27a6: 0x3000, 0x27a7: 0x3000, 0x27a8: 0x3000, 0x27a9: 0x3000, - 0x27aa: 0x3000, 0x27ab: 0x3000, 0x27ac: 0x3000, 0x27ad: 0x3000, 0x27ae: 0x3000, 0x27af: 0x3000, - 0x27b0: 0x3000, 0x27b1: 0x3000, 0x27b2: 0x3000, 0x27b3: 0x3000, 0x27b4: 0x3000, 0x27b5: 0x3000, - 0x27b6: 0x3000, 0x27b7: 0x3000, 0x27b8: 0x3000, 0x27b9: 0x3000, 0x27ba: 0x3000, 0x27bb: 0x3000, - 0x27bc: 0x3000, 0x27bd: 0x3000, 0x27be: 0x3000, 0x27bf: 0x3000, + 0x2782: 0x00e6, 0x2783: 0x00e6, 0x2784: 0x00e6, // Block 0x9f, offset 0x27c0 0x27c0: 0x3000, 0x27c1: 0x3000, 0x27c2: 0x3000, 0x27c3: 0x3000, 0x27c4: 0x3000, 0x27c5: 0x3000, 0x27c6: 0x3000, 0x27c7: 0x3000, 0x27c8: 0x3000, 0x27c9: 0x3000, 0x27ca: 0x3000, 0x27cb: 0x3000, 0x27cc: 0x3000, 0x27cd: 0x3000, 0x27ce: 0x3000, 0x27cf: 0x3000, 0x27d0: 0x3000, 0x27d1: 0x3000, - 0x27d2: 0x3000, 0x27d3: 0x3000, 0x27d4: 0x3000, 0x27d5: 0x3000, 0x27d6: 0x3000, 0x27d7: 0x3000, - 0x27d8: 0x3000, 0x27d9: 0x3000, 0x27da: 0x3000, 0x27db: 0x3000, 0x27dc: 0x3000, - 0x27de: 0x3000, 0x27df: 0x3000, 0x27e2: 0x3000, - 0x27e5: 0x3000, 0x27e6: 0x3000, 0x27e9: 0x3000, - 0x27ea: 0x3000, 0x27eb: 0x3000, 0x27ec: 0x3000, 0x27ee: 0x3000, 0x27ef: 0x3000, + 0x27d2: 0x3000, 0x27d3: 0x3000, 0x27d4: 0x3000, 0x27d6: 0x3000, 0x27d7: 0x3000, + 0x27d8: 0x3000, 0x27d9: 0x3000, 0x27da: 0x3000, 0x27db: 0x3000, 0x27dc: 0x3000, 0x27dd: 0x3000, + 0x27de: 0x3000, 0x27df: 0x3000, 0x27e0: 0x3000, 0x27e1: 0x3000, 0x27e2: 0x3000, 0x27e3: 0x3000, + 0x27e4: 0x3000, 0x27e5: 0x3000, 0x27e6: 0x3000, 0x27e7: 0x3000, 0x27e8: 0x3000, 0x27e9: 0x3000, + 0x27ea: 0x3000, 0x27eb: 0x3000, 0x27ec: 0x3000, 0x27ed: 0x3000, 0x27ee: 0x3000, 0x27ef: 0x3000, 0x27f0: 0x3000, 0x27f1: 0x3000, 0x27f2: 0x3000, 0x27f3: 0x3000, 0x27f4: 0x3000, 0x27f5: 0x3000, - 0x27f6: 0x3000, 0x27f7: 0x3000, 0x27f8: 0x3000, 0x27f9: 0x3000, 0x27fb: 0x3000, - 0x27fd: 0x3000, 0x27fe: 0x3000, 0x27ff: 0x3000, + 0x27f6: 0x3000, 0x27f7: 0x3000, 0x27f8: 0x3000, 0x27f9: 0x3000, 0x27fa: 0x3000, 0x27fb: 0x3000, + 0x27fc: 0x3000, 0x27fd: 0x3000, 0x27fe: 0x3000, 0x27ff: 0x3000, // Block 0xa0, offset 0x2800 - 0x2800: 0x3000, 0x2801: 0x3000, 0x2802: 0x3000, 0x2803: 0x3000, 0x2805: 0x3000, + 0x2800: 0x3000, 0x2801: 0x3000, 0x2802: 0x3000, 0x2803: 0x3000, 0x2804: 0x3000, 0x2805: 0x3000, 0x2806: 0x3000, 0x2807: 0x3000, 0x2808: 0x3000, 0x2809: 0x3000, 0x280a: 0x3000, 0x280b: 0x3000, 0x280c: 0x3000, 0x280d: 0x3000, 0x280e: 0x3000, 0x280f: 0x3000, 0x2810: 0x3000, 0x2811: 0x3000, 0x2812: 0x3000, 0x2813: 0x3000, 0x2814: 0x3000, 0x2815: 0x3000, 0x2816: 0x3000, 0x2817: 0x3000, - 0x2818: 0x3000, 0x2819: 0x3000, 0x281a: 0x3000, 0x281b: 0x3000, 0x281c: 0x3000, 0x281d: 0x3000, - 0x281e: 0x3000, 0x281f: 0x3000, 0x2820: 0x3000, 0x2821: 0x3000, 0x2822: 0x3000, 0x2823: 0x3000, - 0x2824: 0x3000, 0x2825: 0x3000, 0x2826: 0x3000, 0x2827: 0x3000, 0x2828: 0x3000, 0x2829: 0x3000, - 0x282a: 0x3000, 0x282b: 0x3000, 0x282c: 0x3000, 0x282d: 0x3000, 0x282e: 0x3000, 0x282f: 0x3000, + 0x2818: 0x3000, 0x2819: 0x3000, 0x281a: 0x3000, 0x281b: 0x3000, 0x281c: 0x3000, + 0x281e: 0x3000, 0x281f: 0x3000, 0x2822: 0x3000, + 0x2825: 0x3000, 0x2826: 0x3000, 0x2829: 0x3000, + 0x282a: 0x3000, 0x282b: 0x3000, 0x282c: 0x3000, 0x282e: 0x3000, 0x282f: 0x3000, 0x2830: 0x3000, 0x2831: 0x3000, 0x2832: 0x3000, 0x2833: 0x3000, 0x2834: 0x3000, 0x2835: 0x3000, - 0x2836: 0x3000, 0x2837: 0x3000, 0x2838: 0x3000, 0x2839: 0x3000, 0x283a: 0x3000, 0x283b: 0x3000, - 0x283c: 0x3000, 0x283d: 0x3000, 0x283e: 0x3000, 0x283f: 0x3000, + 0x2836: 0x3000, 0x2837: 0x3000, 0x2838: 0x3000, 0x2839: 0x3000, 0x283b: 0x3000, + 0x283d: 0x3000, 0x283e: 0x3000, 0x283f: 0x3000, // Block 0xa1, offset 0x2840 - 0x2840: 0x3000, 0x2841: 0x3000, 0x2842: 0x3000, 0x2843: 0x3000, 0x2844: 0x3000, 0x2845: 0x3000, - 0x2847: 0x3000, 0x2848: 0x3000, 0x2849: 0x3000, 0x284a: 0x3000, - 0x284d: 0x3000, 0x284e: 0x3000, 0x284f: 0x3000, 0x2850: 0x3000, 0x2851: 0x3000, - 0x2852: 0x3000, 0x2853: 0x3000, 0x2854: 0x3000, 0x2856: 0x3000, 0x2857: 0x3000, - 0x2858: 0x3000, 0x2859: 0x3000, 0x285a: 0x3000, 0x285b: 0x3000, 0x285c: 0x3000, + 0x2840: 0x3000, 0x2841: 0x3000, 0x2842: 0x3000, 0x2843: 0x3000, 0x2845: 0x3000, + 0x2846: 0x3000, 0x2847: 0x3000, 0x2848: 0x3000, 0x2849: 0x3000, 0x284a: 0x3000, 0x284b: 0x3000, + 0x284c: 0x3000, 0x284d: 0x3000, 0x284e: 0x3000, 0x284f: 0x3000, 0x2850: 0x3000, 0x2851: 0x3000, + 0x2852: 0x3000, 0x2853: 0x3000, 0x2854: 0x3000, 0x2855: 0x3000, 0x2856: 0x3000, 0x2857: 0x3000, + 0x2858: 0x3000, 0x2859: 0x3000, 0x285a: 0x3000, 0x285b: 0x3000, 0x285c: 0x3000, 0x285d: 0x3000, 0x285e: 0x3000, 0x285f: 0x3000, 0x2860: 0x3000, 0x2861: 0x3000, 0x2862: 0x3000, 0x2863: 0x3000, 0x2864: 0x3000, 0x2865: 0x3000, 0x2866: 0x3000, 0x2867: 0x3000, 0x2868: 0x3000, 0x2869: 0x3000, 0x286a: 0x3000, 0x286b: 0x3000, 0x286c: 0x3000, 0x286d: 0x3000, 0x286e: 0x3000, 0x286f: 0x3000, 0x2870: 0x3000, 0x2871: 0x3000, 0x2872: 0x3000, 0x2873: 0x3000, 0x2874: 0x3000, 0x2875: 0x3000, - 0x2876: 0x3000, 0x2877: 0x3000, 0x2878: 0x3000, 0x2879: 0x3000, 0x287b: 0x3000, - 0x287c: 0x3000, 0x287d: 0x3000, 0x287e: 0x3000, + 0x2876: 0x3000, 0x2877: 0x3000, 0x2878: 0x3000, 0x2879: 0x3000, 0x287a: 0x3000, 0x287b: 0x3000, + 0x287c: 0x3000, 0x287d: 0x3000, 0x287e: 0x3000, 0x287f: 0x3000, // Block 0xa2, offset 0x2880 - 0x2880: 0x3000, 0x2881: 0x3000, 0x2882: 0x3000, 0x2883: 0x3000, 0x2884: 0x3000, - 0x2886: 0x3000, 0x288a: 0x3000, 0x288b: 0x3000, - 0x288c: 0x3000, 0x288d: 0x3000, 0x288e: 0x3000, 0x288f: 0x3000, 0x2890: 0x3000, - 0x2892: 0x3000, 0x2893: 0x3000, 0x2894: 0x3000, 0x2895: 0x3000, 0x2896: 0x3000, 0x2897: 0x3000, - 0x2898: 0x3000, 0x2899: 0x3000, 0x289a: 0x3000, 0x289b: 0x3000, 0x289c: 0x3000, 0x289d: 0x3000, + 0x2880: 0x3000, 0x2881: 0x3000, 0x2882: 0x3000, 0x2883: 0x3000, 0x2884: 0x3000, 0x2885: 0x3000, + 0x2887: 0x3000, 0x2888: 0x3000, 0x2889: 0x3000, 0x288a: 0x3000, + 0x288d: 0x3000, 0x288e: 0x3000, 0x288f: 0x3000, 0x2890: 0x3000, 0x2891: 0x3000, + 0x2892: 0x3000, 0x2893: 0x3000, 0x2894: 0x3000, 0x2896: 0x3000, 0x2897: 0x3000, + 0x2898: 0x3000, 0x2899: 0x3000, 0x289a: 0x3000, 0x289b: 0x3000, 0x289c: 0x3000, 0x289e: 0x3000, 0x289f: 0x3000, 0x28a0: 0x3000, 0x28a1: 0x3000, 0x28a2: 0x3000, 0x28a3: 0x3000, 0x28a4: 0x3000, 0x28a5: 0x3000, 0x28a6: 0x3000, 0x28a7: 0x3000, 0x28a8: 0x3000, 0x28a9: 0x3000, 0x28aa: 0x3000, 0x28ab: 0x3000, 0x28ac: 0x3000, 0x28ad: 0x3000, 0x28ae: 0x3000, 0x28af: 0x3000, 0x28b0: 0x3000, 0x28b1: 0x3000, 0x28b2: 0x3000, 0x28b3: 0x3000, 0x28b4: 0x3000, 0x28b5: 0x3000, - 0x28b6: 0x3000, 0x28b7: 0x3000, 0x28b8: 0x3000, 0x28b9: 0x3000, 0x28ba: 0x3000, 0x28bb: 0x3000, - 0x28bc: 0x3000, 0x28bd: 0x3000, 0x28be: 0x3000, 0x28bf: 0x3000, + 0x28b6: 0x3000, 0x28b7: 0x3000, 0x28b8: 0x3000, 0x28b9: 0x3000, 0x28bb: 0x3000, + 0x28bc: 0x3000, 0x28bd: 0x3000, 0x28be: 0x3000, // Block 0xa3, offset 0x28c0 - 0x28c0: 0x3000, 0x28c1: 0x3000, 0x28c2: 0x3000, 0x28c3: 0x3000, 0x28c4: 0x3000, 0x28c5: 0x3000, - 0x28c6: 0x3000, 0x28c7: 0x3000, 0x28c8: 0x3000, 0x28c9: 0x3000, 0x28ca: 0x3000, 0x28cb: 0x3000, - 0x28cc: 0x3000, 0x28cd: 0x3000, 0x28ce: 0x3000, 0x28cf: 0x3000, 0x28d0: 0x3000, 0x28d1: 0x3000, + 0x28c0: 0x3000, 0x28c1: 0x3000, 0x28c2: 0x3000, 0x28c3: 0x3000, 0x28c4: 0x3000, + 0x28c6: 0x3000, 0x28ca: 0x3000, 0x28cb: 0x3000, + 0x28cc: 0x3000, 0x28cd: 0x3000, 0x28ce: 0x3000, 0x28cf: 0x3000, 0x28d0: 0x3000, 0x28d2: 0x3000, 0x28d3: 0x3000, 0x28d4: 0x3000, 0x28d5: 0x3000, 0x28d6: 0x3000, 0x28d7: 0x3000, 0x28d8: 0x3000, 0x28d9: 0x3000, 0x28da: 0x3000, 0x28db: 0x3000, 0x28dc: 0x3000, 0x28dd: 0x3000, 0x28de: 0x3000, 0x28df: 0x3000, 0x28e0: 0x3000, 0x28e1: 0x3000, 0x28e2: 0x3000, 0x28e3: 0x3000, - 0x28e4: 0x3000, 0x28e5: 0x3000, 0x28e8: 0x3000, 0x28e9: 0x3000, + 0x28e4: 0x3000, 0x28e5: 0x3000, 0x28e6: 0x3000, 0x28e7: 0x3000, 0x28e8: 0x3000, 0x28e9: 0x3000, 0x28ea: 0x3000, 0x28eb: 0x3000, 0x28ec: 0x3000, 0x28ed: 0x3000, 0x28ee: 0x3000, 0x28ef: 0x3000, 0x28f0: 0x3000, 0x28f1: 0x3000, 0x28f2: 0x3000, 0x28f3: 0x3000, 0x28f4: 0x3000, 0x28f5: 0x3000, 0x28f6: 0x3000, 0x28f7: 0x3000, 0x28f8: 0x3000, 0x28f9: 0x3000, 0x28fa: 0x3000, 0x28fb: 0x3000, @@ -6442,53 +6443,65 @@ var charInfoValues = [10944]uint16{ // Block 0xa4, offset 0x2900 0x2900: 0x3000, 0x2901: 0x3000, 0x2902: 0x3000, 0x2903: 0x3000, 0x2904: 0x3000, 0x2905: 0x3000, 0x2906: 0x3000, 0x2907: 0x3000, 0x2908: 0x3000, 0x2909: 0x3000, 0x290a: 0x3000, 0x290b: 0x3000, - 0x290e: 0x3000, 0x290f: 0x3000, 0x2910: 0x3000, 0x2911: 0x3000, + 0x290c: 0x3000, 0x290d: 0x3000, 0x290e: 0x3000, 0x290f: 0x3000, 0x2910: 0x3000, 0x2911: 0x3000, 0x2912: 0x3000, 0x2913: 0x3000, 0x2914: 0x3000, 0x2915: 0x3000, 0x2916: 0x3000, 0x2917: 0x3000, 0x2918: 0x3000, 0x2919: 0x3000, 0x291a: 0x3000, 0x291b: 0x3000, 0x291c: 0x3000, 0x291d: 0x3000, 0x291e: 0x3000, 0x291f: 0x3000, 0x2920: 0x3000, 0x2921: 0x3000, 0x2922: 0x3000, 0x2923: 0x3000, - 0x2924: 0x3000, 0x2925: 0x3000, 0x2926: 0x3000, 0x2927: 0x3000, 0x2928: 0x3000, 0x2929: 0x3000, + 0x2924: 0x3000, 0x2925: 0x3000, 0x2928: 0x3000, 0x2929: 0x3000, 0x292a: 0x3000, 0x292b: 0x3000, 0x292c: 0x3000, 0x292d: 0x3000, 0x292e: 0x3000, 0x292f: 0x3000, 0x2930: 0x3000, 0x2931: 0x3000, 0x2932: 0x3000, 0x2933: 0x3000, 0x2934: 0x3000, 0x2935: 0x3000, 0x2936: 0x3000, 0x2937: 0x3000, 0x2938: 0x3000, 0x2939: 0x3000, 0x293a: 0x3000, 0x293b: 0x3000, 0x293c: 0x3000, 0x293d: 0x3000, 0x293e: 0x3000, 0x293f: 0x3000, // Block 0xa5, offset 0x2940 0x2940: 0x3000, 0x2941: 0x3000, 0x2942: 0x3000, 0x2943: 0x3000, 0x2944: 0x3000, 0x2945: 0x3000, - 0x2946: 0x3000, 0x2947: 0x3000, 0x2948: 0x3000, 0x2949: 0x3000, 0x294a: 0x3000, - 0x2950: 0x3000, 0x2951: 0x3000, + 0x2946: 0x3000, 0x2947: 0x3000, 0x2948: 0x3000, 0x2949: 0x3000, 0x294a: 0x3000, 0x294b: 0x3000, + 0x294e: 0x3000, 0x294f: 0x3000, 0x2950: 0x3000, 0x2951: 0x3000, 0x2952: 0x3000, 0x2953: 0x3000, 0x2954: 0x3000, 0x2955: 0x3000, 0x2956: 0x3000, 0x2957: 0x3000, 0x2958: 0x3000, 0x2959: 0x3000, 0x295a: 0x3000, 0x295b: 0x3000, 0x295c: 0x3000, 0x295d: 0x3000, 0x295e: 0x3000, 0x295f: 0x3000, 0x2960: 0x3000, 0x2961: 0x3000, 0x2962: 0x3000, 0x2963: 0x3000, 0x2964: 0x3000, 0x2965: 0x3000, 0x2966: 0x3000, 0x2967: 0x3000, 0x2968: 0x3000, 0x2969: 0x3000, - 0x296a: 0x3000, 0x296b: 0x3000, 0x296c: 0x3000, 0x296d: 0x3000, 0x296e: 0x3000, + 0x296a: 0x3000, 0x296b: 0x3000, 0x296c: 0x3000, 0x296d: 0x3000, 0x296e: 0x3000, 0x296f: 0x3000, 0x2970: 0x3000, 0x2971: 0x3000, 0x2972: 0x3000, 0x2973: 0x3000, 0x2974: 0x3000, 0x2975: 0x3000, 0x2976: 0x3000, 0x2977: 0x3000, 0x2978: 0x3000, 0x2979: 0x3000, 0x297a: 0x3000, 0x297b: 0x3000, 0x297c: 0x3000, 0x297d: 0x3000, 0x297e: 0x3000, 0x297f: 0x3000, // Block 0xa6, offset 0x2980 0x2980: 0x3000, 0x2981: 0x3000, 0x2982: 0x3000, 0x2983: 0x3000, 0x2984: 0x3000, 0x2985: 0x3000, - 0x2986: 0x3000, 0x2987: 0x3000, 0x2988: 0x3000, 0x2989: 0x3000, 0x298a: 0x3000, 0x298b: 0x3000, - 0x298c: 0x3000, 0x298d: 0x3000, 0x298e: 0x3000, 0x298f: 0x3000, + 0x2986: 0x3000, 0x2987: 0x3000, 0x2988: 0x3000, 0x2989: 0x3000, 0x298a: 0x3000, + 0x2990: 0x3000, 0x2991: 0x3000, + 0x2992: 0x3000, 0x2993: 0x3000, 0x2994: 0x3000, 0x2995: 0x3000, 0x2996: 0x3000, 0x2997: 0x3000, + 0x2998: 0x3000, 0x2999: 0x3000, 0x299a: 0x3000, 0x299b: 0x3000, 0x299c: 0x3000, 0x299d: 0x3000, + 0x299e: 0x3000, 0x299f: 0x3000, 0x29a0: 0x3000, 0x29a1: 0x3000, 0x29a2: 0x3000, 0x29a3: 0x3000, + 0x29a4: 0x3000, 0x29a5: 0x3000, 0x29a6: 0x3000, 0x29a7: 0x3000, 0x29a8: 0x3000, 0x29a9: 0x3000, + 0x29aa: 0x3000, 0x29ab: 0x3000, 0x29ac: 0x3000, 0x29ad: 0x3000, 0x29ae: 0x3000, + 0x29b0: 0x3000, 0x29b1: 0x3000, 0x29b2: 0x3000, 0x29b3: 0x3000, 0x29b4: 0x3000, 0x29b5: 0x3000, + 0x29b6: 0x3000, 0x29b7: 0x3000, 0x29b8: 0x3000, 0x29b9: 0x3000, 0x29ba: 0x3000, 0x29bb: 0x3000, + 0x29bc: 0x3000, 0x29bd: 0x3000, 0x29be: 0x3000, 0x29bf: 0x3000, // Block 0xa7, offset 0x29c0 - 0x29d0: 0x3000, + 0x29c0: 0x3000, 0x29c1: 0x3000, 0x29c2: 0x3000, 0x29c3: 0x3000, 0x29c4: 0x3000, 0x29c5: 0x3000, + 0x29c6: 0x3000, 0x29c7: 0x3000, 0x29c8: 0x3000, 0x29c9: 0x3000, 0x29ca: 0x3000, 0x29cb: 0x3000, + 0x29cc: 0x3000, 0x29cd: 0x3000, 0x29ce: 0x3000, 0x29cf: 0x3000, // Block 0xa8, offset 0x2a00 - 0x2a00: 0x3000, 0x2a01: 0x3000, 0x2a02: 0x3000, - 0x2a10: 0x3000, 0x2a11: 0x3000, - 0x2a12: 0x3000, 0x2a13: 0x3000, 0x2a14: 0x3000, 0x2a15: 0x3000, 0x2a16: 0x3000, 0x2a17: 0x3000, - 0x2a18: 0x3000, 0x2a19: 0x3000, 0x2a1a: 0x3000, 0x2a1b: 0x3000, 0x2a1c: 0x3000, 0x2a1d: 0x3000, - 0x2a1e: 0x3000, 0x2a1f: 0x3000, 0x2a20: 0x3000, 0x2a21: 0x3000, 0x2a22: 0x3000, 0x2a23: 0x3000, - 0x2a24: 0x3000, 0x2a25: 0x3000, 0x2a26: 0x3000, 0x2a27: 0x3000, 0x2a28: 0x3000, 0x2a29: 0x3000, - 0x2a2a: 0x3000, 0x2a2b: 0x3000, 0x2a2c: 0x3000, 0x2a2d: 0x3000, 0x2a2e: 0x3000, 0x2a2f: 0x3000, - 0x2a30: 0x3000, 0x2a31: 0x3000, 0x2a32: 0x3000, 0x2a33: 0x3000, 0x2a34: 0x3000, 0x2a35: 0x3000, - 0x2a36: 0x3000, 0x2a37: 0x3000, 0x2a38: 0x3000, 0x2a39: 0x3000, 0x2a3a: 0x3000, + 0x2a10: 0x3000, // Block 0xa9, offset 0x2a40 - 0x2a40: 0x3000, 0x2a41: 0x3000, 0x2a42: 0x3000, 0x2a43: 0x3000, 0x2a44: 0x3000, 0x2a45: 0x3000, - 0x2a46: 0x3000, 0x2a47: 0x3000, 0x2a48: 0x3000, + 0x2a40: 0x3000, 0x2a41: 0x3000, 0x2a42: 0x3000, 0x2a50: 0x3000, 0x2a51: 0x3000, + 0x2a52: 0x3000, 0x2a53: 0x3000, 0x2a54: 0x3000, 0x2a55: 0x3000, 0x2a56: 0x3000, 0x2a57: 0x3000, + 0x2a58: 0x3000, 0x2a59: 0x3000, 0x2a5a: 0x3000, 0x2a5b: 0x3000, 0x2a5c: 0x3000, 0x2a5d: 0x3000, + 0x2a5e: 0x3000, 0x2a5f: 0x3000, 0x2a60: 0x3000, 0x2a61: 0x3000, 0x2a62: 0x3000, 0x2a63: 0x3000, + 0x2a64: 0x3000, 0x2a65: 0x3000, 0x2a66: 0x3000, 0x2a67: 0x3000, 0x2a68: 0x3000, 0x2a69: 0x3000, + 0x2a6a: 0x3000, 0x2a6b: 0x3000, 0x2a6c: 0x3000, 0x2a6d: 0x3000, 0x2a6e: 0x3000, 0x2a6f: 0x3000, + 0x2a70: 0x3000, 0x2a71: 0x3000, 0x2a72: 0x3000, 0x2a73: 0x3000, 0x2a74: 0x3000, 0x2a75: 0x3000, + 0x2a76: 0x3000, 0x2a77: 0x3000, 0x2a78: 0x3000, 0x2a79: 0x3000, 0x2a7a: 0x3000, // Block 0xaa, offset 0x2a80 - 0x2a80: 0x3300, 0x2a81: 0x3300, 0x2a82: 0x3300, 0x2a83: 0x3300, 0x2a84: 0x3300, 0x2a85: 0x3300, - 0x2a86: 0x3300, 0x2a87: 0x3300, 0x2a88: 0x3300, 0x2a89: 0x3300, 0x2a8a: 0x3300, 0x2a8b: 0x3300, - 0x2a8c: 0x3300, 0x2a8d: 0x3300, 0x2a8e: 0x3300, 0x2a8f: 0x3300, 0x2a90: 0x3300, 0x2a91: 0x3300, - 0x2a92: 0x3300, 0x2a93: 0x3300, 0x2a94: 0x3300, 0x2a95: 0x3300, 0x2a96: 0x3300, 0x2a97: 0x3300, - 0x2a98: 0x3300, 0x2a99: 0x3300, 0x2a9a: 0x3300, 0x2a9b: 0x3300, 0x2a9c: 0x3300, 0x2a9d: 0x3300, + 0x2a80: 0x3000, 0x2a81: 0x3000, 0x2a82: 0x3000, 0x2a83: 0x3000, 0x2a84: 0x3000, 0x2a85: 0x3000, + 0x2a86: 0x3000, 0x2a87: 0x3000, 0x2a88: 0x3000, + 0x2a90: 0x3000, 0x2a91: 0x3000, + // Block 0xab, offset 0x2ac0 + 0x2ac0: 0x3300, 0x2ac1: 0x3300, 0x2ac2: 0x3300, 0x2ac3: 0x3300, 0x2ac4: 0x3300, 0x2ac5: 0x3300, + 0x2ac6: 0x3300, 0x2ac7: 0x3300, 0x2ac8: 0x3300, 0x2ac9: 0x3300, 0x2aca: 0x3300, 0x2acb: 0x3300, + 0x2acc: 0x3300, 0x2acd: 0x3300, 0x2ace: 0x3300, 0x2acf: 0x3300, 0x2ad0: 0x3300, 0x2ad1: 0x3300, + 0x2ad2: 0x3300, 0x2ad3: 0x3300, 0x2ad4: 0x3300, 0x2ad5: 0x3300, 0x2ad6: 0x3300, 0x2ad7: 0x3300, + 0x2ad8: 0x3300, 0x2ad9: 0x3300, 0x2ada: 0x3300, 0x2adb: 0x3300, 0x2adc: 0x3300, 0x2add: 0x3300, } // charInfoLookup: 1152 bytes @@ -6511,64 +6524,64 @@ var charInfoLookup = [1152]uint8{ 0x131: 0x29, 0x132: 0x2a, 0x133: 0x2b, 0x134: 0x2c, 0x135: 0x28, 0x137: 0x2d, 0x138: 0x2e, 0x139: 0x2f, 0x13a: 0x30, 0x13b: 0x31, 0x13c: 0x32, 0x13d: 0x33, 0x13e: 0x34, 0x13f: 0x35, // Block 0x5, offset 0x140 - 0x140: 0x36, 0x142: 0x37, 0x143: 0x38, 0x145: 0x39, 0x146: 0x3a, 0x147: 0x3b, - 0x14d: 0x3c, - 0x15c: 0x3d, 0x15f: 0x3e, - 0x162: 0x3f, 0x164: 0x40, - 0x168: 0x41, 0x169: 0x42, 0x16c: 0x43, 0x16d: 0x44, 0x16e: 0x45, 0x16f: 0x46, - 0x170: 0x47, 0x173: 0x48, 0x174: 0x49, 0x175: 0x4a, 0x176: 0x4b, 0x177: 0x4c, - 0x178: 0x4d, 0x179: 0x4e, 0x17a: 0x4f, 0x17b: 0x50, 0x17c: 0x51, 0x17d: 0x52, 0x17e: 0x53, 0x17f: 0x54, + 0x140: 0x36, 0x142: 0x37, 0x143: 0x38, 0x144: 0x39, 0x145: 0x3a, 0x146: 0x3b, 0x147: 0x3c, + 0x14d: 0x3d, + 0x15c: 0x3e, 0x15f: 0x3f, + 0x162: 0x40, 0x164: 0x41, + 0x168: 0x42, 0x169: 0x43, 0x16c: 0x44, 0x16d: 0x45, 0x16e: 0x46, 0x16f: 0x47, + 0x170: 0x48, 0x173: 0x49, 0x174: 0x4a, 0x175: 0x4b, 0x176: 0x4c, 0x177: 0x4d, + 0x178: 0x4e, 0x179: 0x4f, 0x17a: 0x50, 0x17b: 0x51, 0x17c: 0x52, 0x17d: 0x53, 0x17e: 0x54, 0x17f: 0x55, // Block 0x6, offset 0x180 - 0x180: 0x55, 0x181: 0x56, 0x182: 0x57, 0x183: 0x58, 0x184: 0x59, 0x185: 0x5a, 0x186: 0x5b, 0x187: 0x5c, - 0x188: 0x5d, 0x189: 0x5e, 0x18a: 0x5f, 0x18b: 0x60, 0x18c: 0x61, - 0x191: 0x62, 0x192: 0x63, 0x193: 0x64, - 0x1a8: 0x65, 0x1a9: 0x66, 0x1ab: 0x67, - 0x1b1: 0x68, 0x1b3: 0x69, 0x1b5: 0x6a, 0x1b7: 0x6b, - 0x1ba: 0x6c, 0x1bb: 0x6d, 0x1bc: 0x63, 0x1bd: 0x63, 0x1be: 0x63, 0x1bf: 0x6e, + 0x180: 0x56, 0x181: 0x57, 0x182: 0x58, 0x183: 0x59, 0x184: 0x5a, 0x185: 0x5b, 0x186: 0x5c, 0x187: 0x5d, + 0x188: 0x5e, 0x189: 0x5f, 0x18a: 0x60, 0x18b: 0x61, 0x18c: 0x62, + 0x191: 0x63, 0x192: 0x64, 0x193: 0x65, + 0x1a8: 0x66, 0x1a9: 0x67, 0x1ab: 0x68, + 0x1b1: 0x69, 0x1b3: 0x6a, 0x1b5: 0x6b, 0x1b7: 0x6c, + 0x1ba: 0x6d, 0x1bb: 0x6e, 0x1bc: 0x64, 0x1bd: 0x64, 0x1be: 0x64, 0x1bf: 0x6f, // Block 0x7, offset 0x1c0 - 0x1c0: 0x6f, 0x1c1: 0x70, 0x1c2: 0x71, 0x1c3: 0x72, 0x1c4: 0x73, 0x1c5: 0x63, 0x1c6: 0x74, - 0x1c8: 0x75, 0x1c9: 0x76, 0x1ca: 0x63, 0x1cb: 0x77, 0x1cc: 0x63, 0x1cd: 0x63, 0x1ce: 0x63, 0x1cf: 0x63, + 0x1c0: 0x70, 0x1c1: 0x71, 0x1c2: 0x72, 0x1c3: 0x73, 0x1c4: 0x74, 0x1c5: 0x64, 0x1c6: 0x75, + 0x1c8: 0x76, 0x1c9: 0x77, 0x1ca: 0x64, 0x1cb: 0x78, 0x1cc: 0x64, 0x1cd: 0x64, 0x1ce: 0x64, 0x1cf: 0x64, // Block 0x8, offset 0x200 - 0x219: 0x78, 0x21b: 0x79, 0x21d: 0x7a, - 0x220: 0x7b, 0x223: 0x7c, 0x224: 0x7d, 0x225: 0x7e, 0x226: 0x7f, 0x227: 0x80, - 0x22a: 0x81, 0x22b: 0x82, 0x22f: 0x83, - 0x230: 0x84, 0x231: 0x84, 0x232: 0x84, 0x233: 0x84, 0x234: 0x84, 0x235: 0x84, 0x236: 0x84, 0x237: 0x84, - 0x238: 0x84, 0x239: 0x84, 0x23a: 0x84, 0x23b: 0x84, 0x23c: 0x84, 0x23d: 0x84, 0x23e: 0x84, 0x23f: 0x84, + 0x219: 0x79, 0x21b: 0x7a, 0x21d: 0x7b, + 0x220: 0x7c, 0x223: 0x7d, 0x224: 0x7e, 0x225: 0x7f, 0x226: 0x80, 0x227: 0x81, + 0x22a: 0x82, 0x22b: 0x83, 0x22f: 0x84, + 0x230: 0x85, 0x231: 0x85, 0x232: 0x85, 0x233: 0x85, 0x234: 0x85, 0x235: 0x85, 0x236: 0x85, 0x237: 0x85, + 0x238: 0x85, 0x239: 0x85, 0x23a: 0x85, 0x23b: 0x85, 0x23c: 0x85, 0x23d: 0x85, 0x23e: 0x85, 0x23f: 0x85, // Block 0x9, offset 0x240 - 0x240: 0x84, 0x241: 0x84, 0x242: 0x84, 0x243: 0x84, 0x244: 0x84, 0x245: 0x84, 0x246: 0x84, 0x247: 0x84, - 0x248: 0x84, 0x249: 0x84, 0x24a: 0x84, 0x24b: 0x84, 0x24c: 0x84, 0x24d: 0x84, 0x24e: 0x84, 0x24f: 0x84, - 0x250: 0x84, 0x251: 0x84, 0x252: 0x84, 0x253: 0x84, 0x254: 0x84, 0x255: 0x84, 0x256: 0x84, 0x257: 0x84, - 0x258: 0x84, 0x259: 0x84, 0x25a: 0x84, 0x25b: 0x84, 0x25c: 0x84, 0x25d: 0x84, 0x25e: 0x84, 0x25f: 0x84, - 0x260: 0x84, 0x261: 0x84, 0x262: 0x84, 0x263: 0x84, 0x264: 0x84, 0x265: 0x84, 0x266: 0x84, 0x267: 0x84, - 0x268: 0x84, 0x269: 0x84, 0x26a: 0x84, 0x26b: 0x84, 0x26c: 0x84, 0x26d: 0x84, 0x26e: 0x84, 0x26f: 0x84, - 0x270: 0x84, 0x271: 0x84, 0x272: 0x84, 0x273: 0x84, 0x274: 0x84, 0x275: 0x84, 0x276: 0x84, 0x277: 0x84, - 0x278: 0x84, 0x279: 0x84, 0x27a: 0x84, 0x27b: 0x84, 0x27c: 0x84, 0x27d: 0x84, 0x27e: 0x84, 0x27f: 0x84, + 0x240: 0x85, 0x241: 0x85, 0x242: 0x85, 0x243: 0x85, 0x244: 0x85, 0x245: 0x85, 0x246: 0x85, 0x247: 0x85, + 0x248: 0x85, 0x249: 0x85, 0x24a: 0x85, 0x24b: 0x85, 0x24c: 0x85, 0x24d: 0x85, 0x24e: 0x85, 0x24f: 0x85, + 0x250: 0x85, 0x251: 0x85, 0x252: 0x85, 0x253: 0x85, 0x254: 0x85, 0x255: 0x85, 0x256: 0x85, 0x257: 0x85, + 0x258: 0x85, 0x259: 0x85, 0x25a: 0x85, 0x25b: 0x85, 0x25c: 0x85, 0x25d: 0x85, 0x25e: 0x85, 0x25f: 0x85, + 0x260: 0x85, 0x261: 0x85, 0x262: 0x85, 0x263: 0x85, 0x264: 0x85, 0x265: 0x85, 0x266: 0x85, 0x267: 0x85, + 0x268: 0x85, 0x269: 0x85, 0x26a: 0x85, 0x26b: 0x85, 0x26c: 0x85, 0x26d: 0x85, 0x26e: 0x85, 0x26f: 0x85, + 0x270: 0x85, 0x271: 0x85, 0x272: 0x85, 0x273: 0x85, 0x274: 0x85, 0x275: 0x85, 0x276: 0x85, 0x277: 0x85, + 0x278: 0x85, 0x279: 0x85, 0x27a: 0x85, 0x27b: 0x85, 0x27c: 0x85, 0x27d: 0x85, 0x27e: 0x85, 0x27f: 0x85, // Block 0xa, offset 0x280 - 0x280: 0x84, 0x281: 0x84, 0x282: 0x84, 0x283: 0x84, 0x284: 0x84, 0x285: 0x84, 0x286: 0x84, 0x287: 0x84, - 0x288: 0x84, 0x289: 0x84, 0x28a: 0x84, 0x28b: 0x84, 0x28c: 0x84, 0x28d: 0x84, 0x28e: 0x84, 0x28f: 0x84, - 0x290: 0x84, 0x291: 0x84, 0x292: 0x84, 0x293: 0x84, 0x294: 0x84, 0x295: 0x84, 0x296: 0x84, 0x297: 0x84, - 0x298: 0x84, 0x299: 0x84, 0x29a: 0x84, 0x29b: 0x84, 0x29c: 0x84, 0x29d: 0x84, 0x29e: 0x85, + 0x280: 0x85, 0x281: 0x85, 0x282: 0x85, 0x283: 0x85, 0x284: 0x85, 0x285: 0x85, 0x286: 0x85, 0x287: 0x85, + 0x288: 0x85, 0x289: 0x85, 0x28a: 0x85, 0x28b: 0x85, 0x28c: 0x85, 0x28d: 0x85, 0x28e: 0x85, 0x28f: 0x85, + 0x290: 0x85, 0x291: 0x85, 0x292: 0x85, 0x293: 0x85, 0x294: 0x85, 0x295: 0x85, 0x296: 0x85, 0x297: 0x85, + 0x298: 0x85, 0x299: 0x85, 0x29a: 0x85, 0x29b: 0x85, 0x29c: 0x85, 0x29d: 0x85, 0x29e: 0x86, // Block 0xb, offset 0x2c0 - 0x2e4: 0x86, 0x2e5: 0x86, 0x2e6: 0x86, 0x2e7: 0x86, - 0x2e8: 0x87, 0x2e9: 0x88, 0x2ea: 0x86, 0x2eb: 0x89, 0x2ec: 0x8a, 0x2ed: 0x8b, 0x2ee: 0x8c, 0x2ef: 0x8d, - 0x2f0: 0x63, 0x2f1: 0x63, 0x2f2: 0x63, 0x2f3: 0x63, 0x2f4: 0x8e, 0x2f5: 0x8f, 0x2f6: 0x90, 0x2f7: 0x91, - 0x2f8: 0x92, 0x2f9: 0x93, 0x2fa: 0x63, 0x2fb: 0x94, 0x2fc: 0x95, 0x2fd: 0x63, 0x2fe: 0x77, 0x2ff: 0x96, + 0x2e4: 0x87, 0x2e5: 0x87, 0x2e6: 0x87, 0x2e7: 0x87, + 0x2e8: 0x88, 0x2e9: 0x89, 0x2ea: 0x87, 0x2eb: 0x8a, 0x2ec: 0x8b, 0x2ed: 0x8c, 0x2ee: 0x8d, 0x2ef: 0x8e, + 0x2f0: 0x64, 0x2f1: 0x64, 0x2f2: 0x64, 0x2f3: 0x64, 0x2f4: 0x8f, 0x2f5: 0x90, 0x2f6: 0x91, 0x2f7: 0x92, + 0x2f8: 0x93, 0x2f9: 0x94, 0x2fa: 0x64, 0x2fb: 0x95, 0x2fc: 0x96, 0x2fd: 0x64, 0x2fe: 0x78, 0x2ff: 0x97, // Block 0xc, offset 0x300 - 0x307: 0x97, - 0x328: 0x98, + 0x307: 0x98, + 0x328: 0x99, // Block 0xd, offset 0x340 - 0x341: 0x7b, 0x342: 0x99, + 0x341: 0x7c, 0x342: 0x9a, // Block 0xe, offset 0x380 - 0x385: 0x9a, 0x386: 0x9b, 0x387: 0x9c, - 0x389: 0x9d, - 0x390: 0x63, 0x391: 0x9e, 0x392: 0x9f, 0x393: 0xa0, 0x394: 0xa1, 0x395: 0xa2, 0x396: 0x63, 0x397: 0x63, - 0x398: 0x63, 0x399: 0x63, 0x39a: 0xa3, 0x39b: 0x63, 0x39c: 0x63, 0x39d: 0x63, 0x39e: 0x63, 0x39f: 0xa4, + 0x385: 0x9b, 0x386: 0x9c, 0x387: 0x9d, + 0x389: 0x9e, + 0x390: 0x64, 0x391: 0x9f, 0x392: 0xa0, 0x393: 0xa1, 0x394: 0xa2, 0x395: 0xa3, 0x396: 0x64, 0x397: 0x64, + 0x398: 0x64, 0x399: 0x64, 0x39a: 0xa4, 0x39b: 0x64, 0x39c: 0x64, 0x39d: 0x64, 0x39e: 0x64, 0x39f: 0xa5, // Block 0xf, offset 0x3c0 - 0x3c4: 0xa5, 0x3c5: 0xa6, 0x3c6: 0xa7, - 0x3c8: 0xa8, 0x3c9: 0xa9, + 0x3c4: 0xa6, 0x3c5: 0xa7, 0x3c6: 0xa8, + 0x3c8: 0xa9, 0x3c9: 0xaa, // Block 0x10, offset 0x400 - 0x420: 0x86, 0x421: 0x86, 0x422: 0x86, 0x423: 0x86, 0x424: 0x86, 0x425: 0x86, 0x426: 0x86, 0x427: 0x86, - 0x428: 0xaa, + 0x420: 0x87, 0x421: 0x87, 0x422: 0x87, 0x423: 0x87, 0x424: 0x87, 0x425: 0x87, 0x426: 0x87, 0x427: 0x87, + 0x428: 0xab, // Block 0x11, offset 0x440 0x450: 0x0c, 0x451: 0x0d, 0x45d: 0x0e, 0x45f: 0x0f, @@ -6577,4 +6590,4 @@ var charInfoLookup = [1152]uint8{ var charInfoTrie = trie{charInfoLookup[:], charInfoValues[:]} -// Total size of tables: 78KB (80234 bytes) +// Total size of tables: 78KB (80362 bytes) diff --git a/src/pkg/exp/norm/trie.go b/src/pkg/exp/norm/trie.go index 6b6540187..edae2c212 100644 --- a/src/pkg/exp/norm/trie.go +++ b/src/pkg/exp/norm/trie.go @@ -86,16 +86,6 @@ func (t *trie) lookup(s []byte) (v uint16, sz int) { } o = uint16(i)<<6 + uint16(c3)&maskx return t.values[o], 4 - case c0 < t6: - if len(s) < 5 { - return 0, 0 - } - return 0, 5 - case c0 < te: - if len(s) < 6 { - return 0, 0 - } - return 0, 6 } // Illegal rune return 0, 1 @@ -162,16 +152,6 @@ func (t *trie) lookupString(s string) (v uint16, sz int) { } o = uint16(i)<<6 + uint16(c3)&maskx return t.values[o], 4 - case c0 < t6: - if len(s) < 5 { - return 0, 0 - } - return 0, 5 - case c0 < te: - if len(s) < 6 { - return 0, 0 - } - return 0, 6 } // Illegal rune return 0, 1 diff --git a/src/pkg/exp/norm/trie_test.go b/src/pkg/exp/norm/trie_test.go index ad87d972b..6a309426e 100644 --- a/src/pkg/exp/norm/trie_test.go +++ b/src/pkg/exp/norm/trie_test.go @@ -32,8 +32,10 @@ var tests = []trietest{ {0, []byte{t2}}, {0, []byte{t3, tx}}, {0, []byte{t4, tx, tx}}, - {0, []byte{t5, tx, tx, tx}}, - {0, []byte{t6, tx, tx, tx, tx}}, + + // we only support UTF-8 up to utf8.UTFMax bytes (4 bytes) + {1, []byte{t5, tx, tx, tx, tx}}, + {1, []byte{t6, tx, tx, tx, tx, tx}}, } func mkUtf8(rune int) ([]byte, int) { diff --git a/src/pkg/exp/regexp/exec.go b/src/pkg/exp/regexp/exec.go index 0670bb9b1..88b16032e 100644 --- a/src/pkg/exp/regexp/exec.go +++ b/src/pkg/exp/regexp/exec.go @@ -90,23 +90,12 @@ func (m *machine) match(i input, pos int) bool { if rune != endOfText { rune1, width1 = i.step(pos + width) } - // TODO: Let caller specify the initial flag setting. - // For now assume pos == 0 is beginning of text and - // pos != 0 is not even beginning of line. - // TODO: Word boundary. var flag syntax.EmptyOp if pos == 0 { - flag = syntax.EmptyBeginText | syntax.EmptyBeginLine - } - - // Update flag using lookahead rune. - if rune1 == '\n' { - flag |= syntax.EmptyEndLine - } - if rune1 == endOfText { - flag |= syntax.EmptyEndText + flag = syntax.EmptyOpContext(-1, rune) + } else { + flag = i.context(pos) } - for { if len(runq.dense) == 0 { if startCond&syntax.EmptyBeginText != 0 && pos != 0 { @@ -134,17 +123,7 @@ func (m *machine) match(i input, pos int) bool { } m.add(runq, uint32(m.p.Start), pos, m.matchcap, flag) } - // TODO: word boundary - flag = 0 - if rune == '\n' { - flag |= syntax.EmptyBeginLine - } - if rune1 == '\n' { - flag |= syntax.EmptyEndLine - } - if rune1 == endOfText { - flag |= syntax.EmptyEndText - } + flag = syntax.EmptyOpContext(rune, rune1) m.step(runq, nextq, pos, pos+width, rune, flag) if width == 0 { break diff --git a/src/pkg/exp/regexp/exec_test.go b/src/pkg/exp/regexp/exec_test.go new file mode 100644 index 000000000..15c4c532a --- /dev/null +++ b/src/pkg/exp/regexp/exec_test.go @@ -0,0 +1,271 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package regexp + +import ( + "bufio" + "compress/gzip" + "fmt" + "os" + "strconv" + "strings" + "testing" + "utf8" +) + +// TestRE2 tests this package's regexp API against test cases +// considered during RE2's exhaustive tests, which run all possible +// regexps over a given set of atoms and operators, up to a given +// complexity, over all possible strings over a given alphabet, +// up to a given size. Rather than try to link with RE2, we read a +// log file containing the test cases and the expected matches. +// The log file, re2.txt, is generated by running 'make exhaustive-log' +// in the open source RE2 distribution. http://code.google.com/p/re2/ +// +// The test file format is a sequence of stanzas like: +// +// strings +// "abc" +// "123x" +// regexps +// "[a-z]+" +// 0-3;0-3 +// -;- +// "([0-9])([0-9])([0-9])" +// -;- +// -;0-3 0-1 1-2 2-3 +// +// The stanza begins by defining a set of strings, quoted +// using Go double-quote syntax, one per line. Then the +// regexps section gives a sequence of regexps to run on +// the strings. In the block that follows a regexp, each line +// gives the semicolon-separated match results of running +// the regexp on the corresponding string. +// Each match result is either a single -, meaning no match, or a +// space-separated sequence of pairs giving the match and +// submatch indices. An unmatched subexpression formats +// its pair as a single - (not illustrated above). For now +// each regexp run produces two match results, one for a +// ``full match'' that restricts the regexp to matching the entire +// string or nothing, and one for a ``partial match'' that gives +// the leftmost first match found in the string. +// +// Lines beginning with # are comments. Lines beginning with +// a capital letter are test names printed during RE2's test suite +// and are echoed into t but otherwise ignored. +// +// At time of writing, re2.txt is 32 MB but compresses to 760 kB, +// so we store re2.txt.gz in the repository and decompress it on the fly. +// +func TestRE2(t *testing.T) { + if testing.Short() { + t.Log("skipping TestRE2 during short test") + return + } + + f, err := os.Open("re2.txt.gz") + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatalf("decompress re2.txt.gz: %v", err) + } + defer gz.Close() + lineno := 0 + r := bufio.NewReader(gz) + var ( + str []string + input []string + inStrings bool + re *Regexp + refull *Regexp + nfail int + ncase int + ) + for { + line, err := r.ReadString('\n') + if err != nil { + if err == os.EOF { + break + } + t.Fatalf("re2.txt:%d: %v", lineno, err) + } + line = line[:len(line)-1] // chop \n + lineno++ + switch { + case line == "": + t.Fatalf("re2.txt:%d: unexpected blank line", lineno) + case line[0] == '#': + continue + case 'A' <= line[0] && line[0] <= 'Z': + // Test name. + t.Logf("%s\n", line) + continue + case line == "strings": + str = str[:0] + inStrings = true + case line == "regexps": + inStrings = false + case line[0] == '"': + q, err := strconv.Unquote(line) + if err != nil { + // Fatal because we'll get out of sync. + t.Fatalf("re2.txt:%d: unquote %s: %v", lineno, line, err) + } + if inStrings { + str = append(str, q) + continue + } + // Is a regexp. + if len(input) != 0 { + t.Fatalf("re2.txt:%d: out of sync: have %d strings left before %#q", lineno, len(input), q) + } + re, err = tryCompile(q) + if err != nil { + if err.String() == "error parsing regexp: invalid escape sequence: `\\C`" { + // We don't and likely never will support \C; keep going. + continue + } + t.Errorf("re2.txt:%d: compile %#q: %v", lineno, q, err) + if nfail++; nfail >= 100 { + t.Fatalf("stopping after %d errors", nfail) + } + continue + } + full := `\A(?:` + q + `)\z` + refull, err = tryCompile(full) + if err != nil { + // Fatal because q worked, so this should always work. + t.Fatalf("re2.txt:%d: compile full %#q: %v", lineno, full, err) + } + input = str + case line[0] == '-' || '0' <= line[0] && line[0] <= '9': + // A sequence of match results. + ncase++ + if re == nil { + // Failed to compile: skip results. + continue + } + if len(input) == 0 { + t.Fatalf("re2.txt:%d: out of sync: no input remaining", lineno) + } + var text string + text, input = input[0], input[1:] + if !isSingleBytes(text) && strings.Contains(re.String(), `\B`) { + // RE2's \B considers every byte position, + // so it sees 'not word boundary' in the + // middle of UTF-8 sequences. This package + // only considers the positions between runes, + // so it disagrees. Skip those cases. + continue + } + res := strings.Split(line, ";") + if len(res) != 2 { + t.Fatalf("re2.txt:%d: have %d test results, want 2", lineno, len(res)) + } + // res[0] is full match + // res[1] is partial match + // Run partial match first; don't bother with full if partial fails. + have := re.FindStringSubmatchIndex(text) + want := parseResult(t, lineno, res[1]) + if !same(have, want) { + t.Errorf("re2.txt:%d: %#q.FindSubmatchIndex(%#q) = %v, want %v", lineno, re, text, have, want) + if nfail++; nfail >= 100 { + t.Fatalf("stopping after %d errors", nfail) + } + continue + } + have = refull.FindStringSubmatchIndex(text) + want = parseResult(t, lineno, res[0]) + if !same(have, want) { + t.Errorf("re2.txt:%d: %#q.FindSubmatchIndex(%#q) = %v, want %v", lineno, refull, text, have, want) + if nfail++; nfail >= 100 { + t.Fatalf("stopping after %d errors", nfail) + } + } + default: + t.Fatalf("re2.txt:%d: out of sync: %s\n", lineno, line) + } + } + if len(input) != 0 { + t.Fatalf("re2.txt:%d: out of sync: have %d strings left at EOF", lineno, len(input)) + } + t.Logf("%d cases tested", ncase) +} + +func isSingleBytes(s string) bool { + for _, c := range s { + if c >= utf8.RuneSelf { + return false + } + } + return true +} + +func tryCompile(s string) (re *Regexp, err os.Error) { + // Protect against panic during Compile. + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + return Compile(s) +} + +func parseResult(t *testing.T, lineno int, res string) []int { + // A single - indicates no match. + if res == "-" { + return nil + } + // Otherwise, a space-separated list of pairs. + n := 1 + for j := 0; j < len(res); j++ { + if res[j] == ' ' { + n++ + } + } + out := make([]int, 2*n) + i := 0 + n = 0 + for j := 0; j <= len(res); j++ { + if j == len(res) || res[j] == ' ' { + // Process a single pair. - means no submatch. + pair := res[i:j] + if pair == "-" { + out[n] = -1 + out[n+1] = -1 + } else { + k := strings.Index(pair, "-") + if k < 0 { + t.Fatalf("re2.txt:%d: invalid pair %s", lineno, pair) + } + lo, err1 := strconv.Atoi(pair[:k]) + hi, err2 := strconv.Atoi(pair[k+1:]) + if err1 != nil || err2 != nil || lo > hi { + t.Fatalf("re2.txt:%d: invalid pair %s", lineno, pair) + } + out[n] = lo + out[n+1] = hi + } + n += 2 + i = j + 1 + } + } + return out +} + +func same(x, y []int) bool { + if len(x) != len(y) { + return false + } + for i, xi := range x { + if xi != y[i] { + return false + } + } + return true +} diff --git a/src/pkg/exp/regexp/find_test.go b/src/pkg/exp/regexp/find_test.go index dddc3484c..6406bb6e6 100644 --- a/src/pkg/exp/regexp/find_test.go +++ b/src/pkg/exp/regexp/find_test.go @@ -80,6 +80,23 @@ var findTests = []FindTest{ {`data`, "daXY data", build(1, 5, 9)}, {`da(.)a$`, "daXY data", build(1, 5, 9, 7, 8)}, {`zx+`, "zzx", build(1, 1, 3)}, + {`ab$`, "abcab", build(1, 3, 5)}, + {`(aa)*$`, "a", build(1, 1, 1, -1, -1)}, + {`(?:.|(?:.a))`, "", nil}, + {`(?:A(?:A|a))`, "Aa", build(1, 0, 2)}, + {`(?:A|(?:A|a))`, "a", build(1, 0, 1)}, + {`(a){0}`, "", build(1, 0, 0, -1, -1)}, + {`(?-s)(?:(?:^).)`, "\n", nil}, + {`(?s)(?:(?:^).)`, "\n", build(1, 0, 1)}, + {`(?:(?:^).)`, "\n", nil}, + {`\b`, "x", build(2, 0, 0, 1, 1)}, + {`\b`, "xx", build(2, 0, 0, 2, 2)}, + {`\b`, "x y", build(4, 0, 0, 1, 1, 2, 2, 3, 3)}, + {`\b`, "xx yy", build(4, 0, 0, 2, 2, 3, 3, 5, 5)}, + {`\B`, "x", nil}, + {`\B`, "xx", build(1, 1, 1)}, + {`\B`, "x y", nil}, + {`\B`, "xx yy", build(2, 1, 1, 4, 4)}, // can backslash-escape any punctuation {`\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_\{\|\}\~`, diff --git a/src/pkg/exp/regexp/re2.txt.gz b/src/pkg/exp/regexp/re2.txt.gz new file mode 100644 index 000000000..2b8c832e5 Binary files /dev/null and b/src/pkg/exp/regexp/re2.txt.gz differ diff --git a/src/pkg/exp/regexp/regexp.go b/src/pkg/exp/regexp/regexp.go index 1b75900f8..11feecd55 100644 --- a/src/pkg/exp/regexp/regexp.go +++ b/src/pkg/exp/regexp/regexp.go @@ -84,6 +84,7 @@ type Regexp struct { prefixComplete bool // prefix is the entire regexp prefixRune int // first rune in prefix cond syntax.EmptyOp // empty-width conditions required at start of match + numSubexp int // cache of machines for running regexp mu sync.Mutex @@ -102,13 +103,16 @@ func Compile(expr string) (*Regexp, os.Error) { if err != nil { return nil, err } + maxCap := re.MaxCap() + re = re.Simplify() prog, err := syntax.Compile(re) if err != nil { return nil, err } regexp := &Regexp{ - expr: expr, - prog: prog, + expr: expr, + prog: prog, + numSubexp: maxCap, } regexp.prefix, regexp.prefixComplete = prog.Prefix() if regexp.prefix != "" { @@ -161,9 +165,7 @@ func MustCompile(str string) *Regexp { // NumSubexp returns the number of parenthesized subexpressions in this Regexp. func (re *Regexp) NumSubexp() int { - // NumCap/2 because captures count ( and ) separately. - // -1 because NumCap counts $0 but NumSubexp does not. - return re.prog.NumCap/2 - 1 + return re.numSubexp } const endOfText = -1 @@ -175,6 +177,7 @@ type input interface { canCheckPrefix() bool // can we look ahead without losing info? hasPrefix(re *Regexp) bool index(re *Regexp, pos int) int + context(pos int) syntax.EmptyOp } // inputString scans a string. @@ -205,6 +208,17 @@ func (i *inputString) index(re *Regexp, pos int) int { return strings.Index(i.str[pos:], re.prefix) } +func (i *inputString) context(pos int) syntax.EmptyOp { + r1, r2 := -1, -1 + if pos > 0 && pos <= len(i.str) { + r1, _ = utf8.DecodeLastRuneInString(i.str[:pos]) + } + if pos < len(i.str) { + r2, _ = utf8.DecodeRuneInString(i.str[pos:]) + } + return syntax.EmptyOpContext(r1, r2) +} + // inputBytes scans a byte slice. type inputBytes struct { str []byte @@ -233,6 +247,17 @@ func (i *inputBytes) index(re *Regexp, pos int) int { return bytes.Index(i.str[pos:], re.prefixBytes) } +func (i *inputBytes) context(pos int) syntax.EmptyOp { + r1, r2 := -1, -1 + if pos > 0 && pos <= len(i.str) { + r1, _ = utf8.DecodeLastRune(i.str[:pos]) + } + if pos < len(i.str) { + r2, _ = utf8.DecodeRune(i.str[pos:]) + } + return syntax.EmptyOpContext(r1, r2) +} + // inputReader scans a RuneReader. type inputReader struct { r io.RuneReader @@ -270,6 +295,10 @@ func (i *inputReader) index(re *Regexp, pos int) int { return -1 } +func (i *inputReader) context(pos int) syntax.EmptyOp { + return 0 +} + // LiteralPrefix returns a literal string that must begin any match // of the regular expression re. It returns the boolean true if the // literal string comprises the entire regular expression. @@ -458,6 +487,23 @@ func QuoteMeta(s string) string { return string(b[0:j]) } +// The number of capture values in the program may correspond +// to fewer capturing expressions than are in the regexp. +// For example, "(a){0}" turns into an empty program, so the +// maximum capture in the program is 0 but we need to return +// an expression for \1. Pad appends -1s to the slice a as needed. +func (re *Regexp) pad(a []int) []int { + if a == nil { + // No match. + return nil + } + n := (1 + re.numSubexp) * 2 + for len(a) < n { + a = append(a, -1) + } + return a +} + // Find matches in slice b if b is non-nil, otherwise find matches in string s. func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { var end int @@ -505,7 +551,7 @@ func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) { prevMatchEnd = matches[1] if accept { - deliver(matches) + deliver(re.pad(matches)) i++ } } @@ -580,9 +626,9 @@ func (re *Regexp) FindSubmatch(b []byte) [][]byte { if a == nil { return nil } - ret := make([][]byte, len(a)/2) + ret := make([][]byte, 1+re.numSubexp) for i := range ret { - if a[2*i] >= 0 { + if 2*i < len(a) && a[2*i] >= 0 { ret[i] = b[a[2*i]:a[2*i+1]] } } @@ -595,7 +641,7 @@ func (re *Regexp) FindSubmatch(b []byte) [][]byte { // in the package comment. // A return value of nil indicates no match. func (re *Regexp) FindSubmatchIndex(b []byte) []int { - return re.doExecute(newInputBytes(b), 0, re.prog.NumCap) + return re.pad(re.doExecute(newInputBytes(b), 0, re.prog.NumCap)) } // FindStringSubmatch returns a slice of strings holding the text of the @@ -608,9 +654,9 @@ func (re *Regexp) FindStringSubmatch(s string) []string { if a == nil { return nil } - ret := make([]string, len(a)/2) + ret := make([]string, 1+re.numSubexp) for i := range ret { - if a[2*i] >= 0 { + if 2*i < len(a) && a[2*i] >= 0 { ret[i] = s[a[2*i]:a[2*i+1]] } } @@ -623,7 +669,7 @@ func (re *Regexp) FindStringSubmatch(s string) []string { // 'Index' descriptions in the package comment. // A return value of nil indicates no match. func (re *Regexp) FindStringSubmatchIndex(s string) []int { - return re.doExecute(newInputString(s), 0, re.prog.NumCap) + return re.pad(re.doExecute(newInputString(s), 0, re.prog.NumCap)) } // FindReaderSubmatchIndex returns a slice holding the index pairs @@ -632,7 +678,7 @@ func (re *Regexp) FindStringSubmatchIndex(s string) []int { // by the 'Submatch' and 'Index' descriptions in the package comment. A // return value of nil indicates no match. func (re *Regexp) FindReaderSubmatchIndex(r io.RuneReader) []int { - return re.doExecute(newInputReader(r), 0, re.prog.NumCap) + return re.pad(re.doExecute(newInputReader(r), 0, re.prog.NumCap)) } const startSize = 10 // The size at which to start a slice in the 'All' routines. diff --git a/src/pkg/exp/regexp/syntax/compile.go b/src/pkg/exp/regexp/syntax/compile.go index 5ea2425c3..6b6d06237 100644 --- a/src/pkg/exp/regexp/syntax/compile.go +++ b/src/pkg/exp/regexp/syntax/compile.go @@ -75,6 +75,7 @@ type compiler struct { } // Compile compiles the regexp into a program to be executed. +// The regexp should have been simplified already (returned from re.Simplify). func Compile(re *Regexp) (*Prog, os.Error) { var c compiler c.init() @@ -90,7 +91,7 @@ func (c *compiler) init() { c.inst(InstFail) } -var anyRuneNotNL = []int{0, '\n' - 1, '\n' - 1, unicode.MaxRune} +var anyRuneNotNL = []int{0, '\n' - 1, '\n' + 1, unicode.MaxRune} var anyRune = []int{0, unicode.MaxRune} func (c *compiler) compile(re *Regexp) frag { @@ -105,7 +106,7 @@ func (c *compiler) compile(re *Regexp) frag { } var f frag for j := range re.Rune { - f1 := c.rune(re.Rune[j : j+1]) + f1 := c.rune(re.Rune[j:j+1], re.Flags) if j == 0 { f = f1 } else { @@ -114,11 +115,11 @@ func (c *compiler) compile(re *Regexp) frag { } return f case OpCharClass: - return c.rune(re.Rune) + return c.rune(re.Rune, re.Flags) case OpAnyCharNotNL: - return c.rune(anyRuneNotNL) + return c.rune(anyRuneNotNL, 0) case OpAnyChar: - return c.rune(anyRune) + return c.rune(anyRune, 0) case OpBeginLine: return c.empty(EmptyBeginLine) case OpEndLine: @@ -261,9 +262,16 @@ func (c *compiler) empty(op EmptyOp) frag { return f } -func (c *compiler) rune(rune []int) frag { +func (c *compiler) rune(rune []int, flags Flags) frag { f := c.inst(InstRune) - c.p.Inst[f.i].Rune = rune + i := &c.p.Inst[f.i] + i.Rune = rune + flags &= FoldCase // only relevant flag is FoldCase + if len(rune) != 1 || unicode.SimpleFold(rune[0]) == rune[0] { + // and sometimes not even that + flags &^= FoldCase + } + i.Arg = uint32(flags) f.out = patchList(f.i << 1) return f } diff --git a/src/pkg/exp/regexp/syntax/parse.go b/src/pkg/exp/regexp/syntax/parse.go index 4eed18268..954a0ad8a 100644 --- a/src/pkg/exp/regexp/syntax/parse.go +++ b/src/pkg/exp/regexp/syntax/parse.go @@ -419,8 +419,7 @@ func (p *parser) factor(sub []*Regexp, flags Flags) []*Regexp { // used or marked for reuse, and the slice space has been reused // for out (len(out) <= start). // - // Invariant: sub[start:i] consists of regexps that all begin - // with str as modified by strflags. + // Invariant: sub[start:i] consists of regexps that all begin with ifirst. var ifirst *Regexp if i < len(sub) { ifirst = p.leadingRegexp(sub[i]) @@ -441,7 +440,6 @@ func (p *parser) factor(sub []*Regexp, flags Flags) []*Regexp { } else { // Construct factored form: prefix(suffix1|suffix2|...) prefix := first - for j := start; j < i; j++ { reuse := j != start // prefix came from sub[start] sub[j] = p.removeLeadingRegexp(sub[j], reuse) @@ -605,8 +603,10 @@ func (p *parser) removeLeadingRegexp(re *Regexp, reuse bool) *Regexp { } return re } - re.Op = OpEmptyMatch - return re + if reuse { + p.reuse(re) + } + return p.newRegexp(OpEmptyMatch) } func literalRegexp(s string, flags Flags) *Regexp { @@ -1053,18 +1053,18 @@ func mergeCharClass(dst, src *Regexp) { case OpCharClass: // src is simpler, so either literal or char class if src.Op == OpLiteral { - dst.Rune = appendRange(dst.Rune, src.Rune[0], src.Rune[0]) + dst.Rune = appendLiteral(dst.Rune, src.Rune[0], src.Flags) } else { dst.Rune = appendClass(dst.Rune, src.Rune) } case OpLiteral: // both literal - if src.Rune[0] == dst.Rune[0] { + if src.Rune[0] == dst.Rune[0] && src.Flags == dst.Flags { break } dst.Op = OpCharClass - dst.Rune = append(dst.Rune, dst.Rune[0]) - dst.Rune = appendRange(dst.Rune, src.Rune[0], src.Rune[0]) + dst.Rune = appendLiteral(dst.Rune[:0], dst.Rune[0], dst.Flags) + dst.Rune = appendLiteral(dst.Rune, src.Rune[0], src.Flags) } } @@ -1544,6 +1544,14 @@ func cleanClass(rp *[]int) []int { return r[:w] } +// appendLiteral returns the result of appending the literal x to the class r. +func appendLiteral(r []int, x int, flags Flags) []int { + if flags&FoldCase != 0 { + return appendFoldedRange(r, x, x) + } + return appendRange(r, x, x) +} + // appendRange returns the result of appending the range lo-hi to the class r. func appendRange(r []int, lo, hi int) []int { // Expand last range or next to last range if it overlaps or abuts. diff --git a/src/pkg/exp/regexp/syntax/parse_test.go b/src/pkg/exp/regexp/syntax/parse_test.go index 779b9afde..a146c89c3 100644 --- a/src/pkg/exp/regexp/syntax/parse_test.go +++ b/src/pkg/exp/regexp/syntax/parse_test.go @@ -162,6 +162,18 @@ var parseTests = []struct { // Factoring. {`abc|abd|aef|bcx|bcy`, `alt{cat{lit{a}alt{cat{lit{b}cc{0x63-0x64}}str{ef}}}cat{str{bc}cc{0x78-0x79}}}`}, {`ax+y|ax+z|ay+w`, `cat{lit{a}alt{cat{plus{lit{x}}cc{0x79-0x7a}}cat{plus{lit{y}}lit{w}}}}`}, + + // Bug fixes. + {`(?:.)`, `dot{}`}, + {`(?:x|(?:xa))`, `cat{lit{x}alt{emp{}lit{a}}}`}, + {`(?:.|(?:.a))`, `cat{dot{}alt{emp{}lit{a}}}`}, + {`(?:A(?:A|a))`, `cat{lit{A}litfold{A}}`}, + {`(?:A|a)`, `litfold{A}`}, + {`A|(?:A|a)`, `litfold{A}`}, + {`(?s).`, `dot{}`}, + {`(?-s).`, `dnl{}`}, + {`(?:(?:^).)`, `cat{bol{}dot{}}`}, + {`(?-s)(?:(?:^).)`, `cat{bol{}dnl{}}`}, } const testFlags = MatchNL | PerlX | UnicodeGroups diff --git a/src/pkg/exp/regexp/syntax/prog.go b/src/pkg/exp/regexp/syntax/prog.go index bf85b720d..d214d70b5 100644 --- a/src/pkg/exp/regexp/syntax/prog.go +++ b/src/pkg/exp/regexp/syntax/prog.go @@ -3,6 +3,7 @@ package syntax import ( "bytes" "strconv" + "unicode" ) // Compiled program. @@ -41,6 +42,41 @@ const ( EmptyNoWordBoundary ) +// EmptyOpContext returns the zero-width assertions +// satisfied at the position between the runes r1 and r2. +// Passing r1 == -1 indicates that the position is +// at the beginning of the text. +// Passing r2 == -1 indicates that the position is +// at the end of the text. +func EmptyOpContext(r1, r2 int) EmptyOp { + var op EmptyOp + if r1 < 0 { + op |= EmptyBeginText | EmptyBeginLine + } + if r1 == '\n' { + op |= EmptyBeginLine + } + if r2 < 0 { + op |= EmptyEndText + } + if r2 == '\n' { + op |= EmptyEndLine + } + if IsWordChar(r1) != IsWordChar(r2) { + op |= EmptyWordBoundary + } else { + op |= EmptyNoWordBoundary + } + return op +} + +// IsWordChar reports whether r is consider a ``word character'' +// during the evaluation of the \b and \B zero-width assertions. +// These assertions are ASCII-only: the word characters are [A-Za-z0-9_]. +func IsWordChar(r int) bool { + return 'A' <= r && r <= 'Z' || 'a' <= r && r <= 'z' || '0' <= r && r <= '9' || r == '_' +} + // An Inst is a single instruction in a regular expression program. type Inst struct { Op InstOp @@ -79,7 +115,7 @@ func (p *Prog) Prefix() (prefix string, complete bool) { // Have prefix; gather characters. var buf bytes.Buffer - for i.Op == InstRune && len(i.Rune) == 1 { + for i.Op == InstRune && len(i.Rune) == 1 && Flags(i.Arg)&FoldCase == 0 { buf.WriteRune(i.Rune[0]) i = p.skipNop(i.Out) } @@ -116,9 +152,19 @@ func (i *Inst) MatchRune(r int) bool { rune := i.Rune // Special case: single-rune slice is from literal string, not char class. - // TODO: Case folding. if len(rune) == 1 { - return r == rune[0] + r0 := rune[0] + if r == r0 { + return true + } + if Flags(i.Arg)&FoldCase != 0 { + for r1 := unicode.SimpleFold(r0); r1 != r0; r1 = unicode.SimpleFold(r1) { + if r == r1 { + return true + } + } + } + return false } // Peek at the first few pairs. @@ -232,6 +278,10 @@ func dumpInst(b *bytes.Buffer, i *Inst) { // shouldn't happen bw(b, "rune ") } - bw(b, "rune ", strconv.QuoteToASCII(string(i.Rune)), " -> ", u32(i.Out)) + bw(b, "rune ", strconv.QuoteToASCII(string(i.Rune))) + if Flags(i.Arg)&FoldCase != 0 { + bw(b, "/i") + } + bw(b, " -> ", u32(i.Out)) } } diff --git a/src/pkg/exp/regexp/syntax/prog_test.go b/src/pkg/exp/regexp/syntax/prog_test.go index 7be4281c2..3fe0c5870 100644 --- a/src/pkg/exp/regexp/syntax/prog_test.go +++ b/src/pkg/exp/regexp/syntax/prog_test.go @@ -76,6 +76,16 @@ var compileTests = []struct { 4 alt -> 3, 6 5* alt -> 1, 3 6 match +`}, + {"A[Aa]", ` 0 fail + 1* rune "A" -> 2 + 2 rune "A"/i -> 3 + 3 match +`}, + {"(?:(?:^).)", ` 0 fail + 1* empty 4 -> 2 + 2 rune "\x00\t\v\U0010ffff" -> 3 + 3 match `}, } diff --git a/src/pkg/exp/regexp/syntax/regexp.go b/src/pkg/exp/regexp/syntax/regexp.go index 00a4addef..d8f51b903 100644 --- a/src/pkg/exp/regexp/syntax/regexp.go +++ b/src/pkg/exp/regexp/syntax/regexp.go @@ -282,3 +282,17 @@ func escape(b *bytes.Buffer, r int, force bool) { b.WriteString(`}`) } } + +// MaxCap walks the regexp to find the maximum capture index. +func (re *Regexp) MaxCap() int { + m := 0 + if re.Op == OpCapture { + m = re.Cap + } + for _, sub := range re.Sub { + if n := sub.MaxCap(); m < n { + m = n + } + } + return m +} diff --git a/src/pkg/exp/template/html/Makefile b/src/pkg/exp/template/html/Makefile index 2f107da11..3a93bebc0 100644 --- a/src/pkg/exp/template/html/Makefile +++ b/src/pkg/exp/template/html/Makefile @@ -6,6 +6,8 @@ include ../../../../Make.inc TARG=exp/template/html GOFILES=\ - escape.go + context.go\ + escape.go\ + js.go\ include ../../../../Make.pkg diff --git a/src/pkg/exp/template/html/context.go b/src/pkg/exp/template/html/context.go index 411006883..428b3d0b3 100644 --- a/src/pkg/exp/template/html/context.go +++ b/src/pkg/exp/template/html/context.go @@ -16,56 +16,77 @@ import ( // http://www.w3.org/TR/html5/the-end.html#parsing-html-fragments // where the context element is null. type context struct { - state state - delim delim + state state + delim delim + urlPart urlPart + jsCtx jsCtx + errLine int + errStr string } -func (c context) String() string { - return fmt.Sprintf("context{state: %s, delim: %s", c.state, c.delim) -} - -// eq is true if the two contexts are identical field-wise. +// eq returns whether two contexts are equal. func (c context) eq(d context) bool { - return c.state == d.state && c.delim == d.delim + return c.state == d.state && c.delim == d.delim && c.urlPart == d.urlPart && c.jsCtx == d.jsCtx && c.errLine == d.errLine && c.errStr == d.errStr } // state describes a high-level HTML parser state. // -// It bounds the top of the element stack, and by extension the HTML -// insertion mode, but also contains state that does not correspond to -// anything in the HTML5 parsing algorithm because a single token -// production in the HTML grammar may contain embedded actions in a template. -// For instance, the quoted HTML attribute produced by +// It bounds the top of the element stack, and by extension the HTML insertion +// mode, but also contains state that does not correspond to anything in the +// HTML5 parsing algorithm because a single token production in the HTML +// grammar may contain embedded actions in a template. For instance, the quoted +// HTML attribute produced by //
// is a single token in HTML's grammar but in a template spans several nodes. type state uint8 const ( - // statePCDATA is parsed character data. An HTML parser is in + // stateText is parsed character data. An HTML parser is in // this state when its parse position is outside an HTML tag, // directive, comment, and special element body. - statePCDATA state = iota + stateText state = iota // stateTag occurs before an HTML attribute or the end of a tag. stateTag - // stateURI occurs inside an HTML attribute whose content is a URI. - stateURI + // stateAttr occurs inside an HTML attribute whose content is text. + stateAttr + // stateURL occurs inside an HTML attribute whose content is a URL. + stateURL + // stateJS occurs inside an event handler or script element. + stateJS + // stateJSDqStr occurs inside a JavaScript double quoted string. + stateJSDqStr + // stateJSSqStr occurs inside a JavaScript single quoted string. + stateJSSqStr + // stateJSRegexp occurs inside a JavaScript regexp literal. + stateJSRegexp + // stateJSBlockCmt occurs inside a JavaScript /* block comment */. + stateJSBlockCmt + // stateJSLineCmt occurs inside a JavaScript // line comment. + stateJSLineCmt // stateError is an infectious error state outside any valid // HTML/CSS/JS construct. stateError ) var stateNames = [...]string{ - statePCDATA: "statePCDATA", - stateTag: "stateTag", - stateURI: "stateURI", - stateError: "stateError", + stateText: "stateText", + stateTag: "stateTag", + stateAttr: "stateAttr", + stateURL: "stateURL", + stateJS: "stateJS", + stateJSDqStr: "stateJSDqStr", + stateJSSqStr: "stateJSSqStr", + stateJSRegexp: "stateJSRegexp", + stateJSBlockCmt: "stateJSBlockCmt", + stateJSLineCmt: "stateJSLineCmt", + stateError: "stateError", } func (s state) String() string { - if uint(s) < uint(len(stateNames)) { + if int(s) < len(stateNames) { return stateNames[s] } - return fmt.Sprintf("illegal state %d", uint(s)) + return fmt.Sprintf("illegal state %d", s) } // delim is the delimiter that will end the current HTML attribute. @@ -91,8 +112,62 @@ var delimNames = [...]string{ } func (d delim) String() string { - if uint(d) < uint(len(delimNames)) { + if int(d) < len(delimNames) { return delimNames[d] } - return fmt.Sprintf("illegal delim %d", uint(d)) + return fmt.Sprintf("illegal delim %d", d) +} + +// urlPart identifies a part in an RFC 3986 hierarchical URL to allow different +// encoding strategies. +type urlPart uint8 + +const ( + // urlPartNone occurs when not in a URL, or possibly at the start: + // ^ in "^http://auth/path?k=v#frag". + urlPartNone urlPart = iota + // urlPartPreQuery occurs in the scheme, authority, or path; between the + // ^s in "h^ttp://auth/path^?k=v#frag". + urlPartPreQuery + // urlPartQueryOrFrag occurs in the query portion between the ^s in + // "http://auth/path?^k=v#frag^". + urlPartQueryOrFrag + // urlPartUnknown occurs due to joining of contexts both before and after + // the query separator. + urlPartUnknown +) + +var urlPartNames = [...]string{ + urlPartNone: "urlPartNone", + urlPartPreQuery: "urlPartPreQuery", + urlPartQueryOrFrag: "urlPartQueryOrFrag", + urlPartUnknown: "urlPartUnknown", +} + +func (u urlPart) String() string { + if int(u) < len(urlPartNames) { + return urlPartNames[u] + } + return fmt.Sprintf("illegal urlPart %d", u) +} + +// jsCtx determines whether a '/' starts a regular expression literal or a +// division operator. +type jsCtx uint8 + +const ( + // jsCtxRegexp occurs where a '/' would start a regexp literal. + jsCtxRegexp jsCtx = iota + // jsCtxDivOp occurs where a '/' would start a division operator. + jsCtxDivOp +) + +func (c jsCtx) String() string { + switch c { + case jsCtxRegexp: + return "jsCtxRegexp" + case jsCtxDivOp: + return "jsCtxDivOp" + } + return fmt.Sprintf("illegal jsCtx %d", c) } diff --git a/src/pkg/exp/template/html/escape.go b/src/pkg/exp/template/html/escape.go index e0e87b98d..0eb8dfec8 100644 --- a/src/pkg/exp/template/html/escape.go +++ b/src/pkg/exp/template/html/escape.go @@ -2,104 +2,638 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package html is a specialization of exp/template that automates the +// Package html is a specialization of template that automates the // construction of safe HTML output. -// At the moment, the escaping is naive. All dynamic content is assumed to be -// plain text interpolated in an HTML PCDATA context. +// INCOMPLETE. package html import ( + "bytes" + "fmt" + "html" + "os" + "strings" "template" "template/parse" ) -// Escape rewrites each action in the template to guarantee the output is +// Escape rewrites each action in the template to guarantee that the output is // HTML-escaped. -func Escape(t *template.Template) { - // If the parser shares trees based on common-subexpression - // joining then we will need to avoid multiply escaping the same action. - escapeListNode(t.Tree.Root) +func Escape(t *template.Template) (*template.Template, os.Error) { + c := escapeList(context{}, t.Tree.Root) + if c.errStr != "" { + return nil, fmt.Errorf("%s:%d: %s", t.Name(), c.errLine, c.errStr) + } + if c.state != stateText { + return nil, fmt.Errorf("%s ends in a non-text context: %v", t.Name(), c) + } + t.Funcs(funcMap) + return t, nil } -// escapeNode dispatches to escape helpers by type. -func escapeNode(node parse.Node) { - switch n := node.(type) { - case *parse.ListNode: - escapeListNode(n) - case *parse.TextNode: - // Nothing to do. +// funcMap maps command names to functions that render their inputs safe. +var funcMap = template.FuncMap{ + "exp_template_html_urlfilter": urlFilter, + "exp_template_html_jsvalescaper": jsValEscaper, + "exp_template_html_jsstrescaper": jsStrEscaper, + "exp_template_html_jsregexpescaper": jsRegexpEscaper, +} + +// escape escapes a template node. +func escape(c context, n parse.Node) context { + switch n := n.(type) { case *parse.ActionNode: - escapeActionNode(n) + return escapeAction(c, n) case *parse.IfNode: - escapeIfNode(n) + return escapeBranch(c, &n.BranchNode, "if") + case *parse.ListNode: + return escapeList(c, n) case *parse.RangeNode: - escapeRangeNode(n) - case *parse.TemplateNode: - // Nothing to do. + return escapeBranch(c, &n.BranchNode, "range") + case *parse.TextNode: + return escapeText(c, n.Text) case *parse.WithNode: - escapeWithNode(n) + return escapeBranch(c, &n.BranchNode, "with") + } + // TODO: handle a *parse.TemplateNode. Should Escape take a *template.Set? + panic("escaping " + n.String() + " is unimplemented") +} + +// escapeAction escapes an action template node. +func escapeAction(c context, n *parse.ActionNode) context { + s := make([]string, 0, 2) + switch c.state { + case stateURL: + switch c.urlPart { + case urlPartNone: + s = append(s, "exp_template_html_urlfilter") + case urlPartQueryOrFrag: + s = append(s, "urlquery") + case urlPartPreQuery: + s = append(s, "html") + case urlPartUnknown: + return context{ + state: stateError, + errLine: n.Line, + errStr: fmt.Sprintf("%s appears in an ambiguous URL context", n), + } + default: + panic(c.urlPart.String()) + } + case stateJS: + s = append(s, "exp_template_html_jsvalescaper") + if c.delim != delimNone { + s = append(s, "html") + } + case stateJSDqStr, stateJSSqStr: + s = append(s, "exp_template_html_jsstrescaper") + case stateJSRegexp: + s = append(s, "exp_template_html_jsregexpescaper") + case stateJSBlockCmt, stateJSLineCmt: + return context{ + state: stateError, + errLine: n.Line, + errStr: fmt.Sprintf("%s appears inside a comment", n), + } default: - panic("handling for " + node.String() + " not implemented") - // TODO: Handle other inner node types. + s = append(s, "html") } + ensurePipelineContains(n.Pipe, s) + return c } -// escapeListNode recursively escapes its input's children. -func escapeListNode(node *parse.ListNode) { - if node == nil { +// ensurePipelineContains ensures that the pipeline has commands with +// the identifiers in s in order. +// If the pipeline already has some of the sanitizers, do not interfere. +// For example, if p is (.X | html) and s is ["escapeJSVal", "html"] then it +// has one matching, "html", and one to insert, "escapeJSVal", to produce +// (.X | escapeJSVal | html). +func ensurePipelineContains(p *parse.PipeNode, s []string) { + if len(s) == 0 { return } - children := node.Nodes - for _, child := range children { - escapeNode(child) + n := len(p.Cmds) + // Find the identifiers at the end of the command chain. + idents := p.Cmds + for i := n - 1; i >= 0; i-- { + if cmd := p.Cmds[i]; len(cmd.Args) != 0 { + if _, ok := cmd.Args[0].(*parse.IdentifierNode); ok { + continue + } + } + idents = p.Cmds[i+1:] } + dups := 0 + for _, id := range idents { + if s[dups] == (id.Args[0].(*parse.IdentifierNode)).Ident { + dups++ + if dups == len(s) { + return + } + } + } + newCmds := make([]*parse.CommandNode, n-len(idents), n+len(s)-dups) + copy(newCmds, p.Cmds) + // Merge existing identifier commands with the sanitizers needed. + for _, id := range idents { + i := indexOfStr((id.Args[0].(*parse.IdentifierNode)).Ident, s) + if i != -1 { + for _, name := range s[:i] { + newCmds = append(newCmds, newIdentCmd(name)) + } + s = s[i+1:] + } + newCmds = append(newCmds, id) + } + // Create any remaining sanitizers. + for _, name := range s { + newCmds = append(newCmds, newIdentCmd(name)) + } + p.Cmds = newCmds } -// escapeActionNode adds a pipeline call to the end that escapes the result -// of the expression before it is interpolated into the template output. -func escapeActionNode(node *parse.ActionNode) { - pipe := node.Pipe +// indexOfStr is the least i such that strs[i] == s or -1 if s is not in strs. +func indexOfStr(s string, strs []string) int { + for i, t := range strs { + if s == t { + return i + } + } + return -1 +} - cmds := pipe.Cmds - nCmds := len(cmds) +// newIdentCmd produces a command containing a single identifier node. +func newIdentCmd(identifier string) *parse.CommandNode { + return &parse.CommandNode{ + NodeType: parse.NodeCommand, + Args: []parse.Node{parse.NewIdentifier(identifier)}, + } +} - // If it already has an escaping command, do not interfere. - if nCmds != 0 { - if lastCmd := cmds[nCmds-1]; len(lastCmd.Args) != 0 { - // TODO: Recognize url and js as escaping functions once - // we have enough context to know whether additional - // escaping is necessary. - if arg, ok := lastCmd.Args[0].(*parse.IdentifierNode); ok && arg.Ident == "html" { - return +// join joins the two contexts of a branch template node. The result is an +// error context if either of the input contexts are error contexts, or if the +// the input contexts differ. +func join(a, b context, line int, nodeName string) context { + if a.state == stateError { + return a + } + if b.state == stateError { + return b + } + if a.eq(b) { + return a + } + + c := a + c.urlPart = b.urlPart + if c.eq(b) { + // The contexts differ only by urlPart. + c.urlPart = urlPartUnknown + return c + } + + return context{ + state: stateError, + errLine: line, + errStr: fmt.Sprintf("{{%s}} branches end in different contexts: %v, %v", nodeName, a, b), + } +} + +// escapeBranch escapes a branch template node: "if", "range" and "with". +func escapeBranch(c context, n *parse.BranchNode, nodeName string) context { + c0 := escapeList(c, n.List) + if nodeName == "range" && c0.state != stateError { + // The "true" branch of a "range" node can execute multiple times. + // We check that executing n.List once results in the same context + // as executing n.List twice. + c0 = join(c0, escapeList(c0, n.List), n.Line, nodeName) + if c0.state == stateError { + // Make clear that this is a problem on loop re-entry + // since developers tend to overlook that branch when + // debugging templates. + c0.errLine = n.Line + c0.errStr = "on range loop re-entry: " + c0.errStr + return c0 + } + } + c1 := escapeList(c, n.ElseList) + return join(c0, c1, n.Line, nodeName) +} + +// escapeList escapes a list template node. +func escapeList(c context, n *parse.ListNode) context { + if n == nil { + return c + } + for _, m := range n.Nodes { + c = escape(c, m) + } + return c +} + +// delimEnds maps each delim to a string of characters that terminate it. +var delimEnds = [...]string{ + delimDoubleQuote: `"`, + delimSingleQuote: "'", + // Determined empirically by running the below in various browsers. + // var div = document.createElement("DIV"); + // for (var i = 0; i < 0x10000; ++i) { + // div.innerHTML = ""; + // if (div.getElementsByTagName("SPAN")[0].title.indexOf("bar") < 0) + // document.write("

U+" + i.toString(16)); + // } + delimSpaceOrTagEnd: " \t\n\f\r>", +} + +// escapeText escapes a text template node. +func escapeText(c context, s []byte) context { + for len(s) > 0 { + if c.delim == delimNone { + c, s = transitionFunc[c.state](c, s) + continue + } + + i := bytes.IndexAny(s, delimEnds[c.delim]) + if i == -1 { + // Remain inside the attribute. + // Decode the value so non-HTML rules can easily handle + //